Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Portions Copyright (C) 1992 Drew Eckhardt
4 */
5#ifndef _LINUX_BLKDEV_H
6#define _LINUX_BLKDEV_H
7
8#include <linux/types.h>
9#include <linux/blk_types.h>
10#include <linux/device.h>
11#include <linux/list.h>
12#include <linux/llist.h>
13#include <linux/minmax.h>
14#include <linux/timer.h>
15#include <linux/workqueue.h>
16#include <linux/completion.h>
17#include <linux/wait.h>
18#include <linux/bio.h>
19#include <linux/gfp.h>
20#include <linux/kdev_t.h>
21#include <linux/rcupdate.h>
22#include <linux/percpu-refcount.h>
23#include <linux/blkzoned.h>
24#include <linux/sched.h>
25#include <linux/sbitmap.h>
26#include <linux/uuid.h>
27#include <linux/xarray.h>
28#include <linux/file.h>
29#include <linux/lockdep.h>
30
31struct module;
32struct request_queue;
33struct elevator_queue;
34struct blk_trace;
35struct request;
36struct sg_io_hdr;
37struct blkcg_gq;
38struct blk_flush_queue;
39struct kiocb;
40struct pr_ops;
41struct rq_qos;
42struct blk_report_zones_args;
43struct blk_queue_stats;
44struct blk_stat_callback;
45struct blk_crypto_profile;
46
47extern const struct device_type disk_type;
48extern const struct device_type part_type;
49extern const struct class block_class;
50
51/*
52 * Maximum number of blkcg policies allowed to be registered concurrently.
53 * Defined here to simplify include dependency.
54 */
55#define BLKCG_MAX_POLS 6
56
57#define DISK_MAX_PARTS 256
58#define DISK_NAME_LEN 32
59
60#define PARTITION_META_INFO_VOLNAMELTH 64
61/*
62 * Enough for the string representation of any kind of UUID plus NULL.
63 * EFI UUID is 36 characters. MSDOS UUID is 11 characters.
64 */
65#define PARTITION_META_INFO_UUIDLTH (UUID_STRING_LEN + 1)
66
67struct partition_meta_info {
68 char uuid[PARTITION_META_INFO_UUIDLTH];
69 u8 volname[PARTITION_META_INFO_VOLNAMELTH];
70};
71
72/**
73 * DOC: genhd capability flags
74 *
75 * ``GENHD_FL_REMOVABLE``: indicates that the block device gives access to
76 * removable media. When set, the device remains present even when media is not
77 * inserted. Shall not be set for devices which are removed entirely when the
78 * media is removed.
79 *
80 * ``GENHD_FL_HIDDEN``: the block device is hidden; it doesn't produce events,
81 * doesn't appear in sysfs, and can't be opened from userspace or using
82 * blkdev_get*. Used for the underlying components of multipath devices.
83 *
84 * ``GENHD_FL_NO_PART``: partition support is disabled. The kernel will not
85 * scan for partitions from add_disk, and users can't add partitions manually.
86 *
87 */
88enum {
89 GENHD_FL_REMOVABLE = 1 << 0,
90 GENHD_FL_HIDDEN = 1 << 1,
91 GENHD_FL_NO_PART = 1 << 2,
92};
93
94enum {
95 DISK_EVENT_MEDIA_CHANGE = 1 << 0, /* media changed */
96 DISK_EVENT_EJECT_REQUEST = 1 << 1, /* eject requested */
97};
98
99enum {
100 /* Poll even if events_poll_msecs is unset */
101 DISK_EVENT_FLAG_POLL = 1 << 0,
102 /* Forward events to udev */
103 DISK_EVENT_FLAG_UEVENT = 1 << 1,
104 /* Block event polling when open for exclusive write */
105 DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE = 1 << 2,
106};
107
108struct disk_events;
109struct badblocks;
110
111enum blk_integrity_checksum {
112 BLK_INTEGRITY_CSUM_NONE = 0,
113 BLK_INTEGRITY_CSUM_IP = 1,
114 BLK_INTEGRITY_CSUM_CRC = 2,
115 BLK_INTEGRITY_CSUM_CRC64 = 3,
116} __packed ;
117
118struct blk_integrity {
119 unsigned char flags;
120 enum blk_integrity_checksum csum_type;
121 unsigned char metadata_size;
122 unsigned char pi_offset;
123 unsigned char interval_exp;
124 unsigned char tag_size;
125 unsigned char pi_tuple_size;
126};
127
128typedef unsigned int __bitwise blk_mode_t;
129
130/* open for reading */
131#define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0))
132/* open for writing */
133#define BLK_OPEN_WRITE ((__force blk_mode_t)(1 << 1))
134/* open exclusively (vs other exclusive openers */
135#define BLK_OPEN_EXCL ((__force blk_mode_t)(1 << 2))
136/* opened with O_NDELAY */
137#define BLK_OPEN_NDELAY ((__force blk_mode_t)(1 << 3))
138/* open for "writes" only for ioctls (specialy hack for floppy.c) */
139#define BLK_OPEN_WRITE_IOCTL ((__force blk_mode_t)(1 << 4))
140/* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */
141#define BLK_OPEN_RESTRICT_WRITES ((__force blk_mode_t)(1 << 5))
142/* return partition scanning errors */
143#define BLK_OPEN_STRICT_SCAN ((__force blk_mode_t)(1 << 6))
144
145struct gendisk {
146 /*
147 * major/first_minor/minors should not be set by any new driver, the
148 * block core will take care of allocating them automatically.
149 */
150 int major;
151 int first_minor;
152 int minors;
153
154 char disk_name[DISK_NAME_LEN]; /* name of major driver */
155
156 unsigned short events; /* supported events */
157 unsigned short event_flags; /* flags related to event processing */
158
159 struct xarray part_tbl;
160 struct block_device *part0;
161
162 const struct block_device_operations *fops;
163 struct request_queue *queue;
164 void *private_data;
165
166 struct bio_set bio_split;
167
168 int flags;
169 unsigned long state;
170#define GD_NEED_PART_SCAN 0
171#define GD_READ_ONLY 1
172#define GD_DEAD 2
173#define GD_NATIVE_CAPACITY 3
174#define GD_ADDED 4
175#define GD_SUPPRESS_PART_SCAN 5
176#define GD_OWNS_QUEUE 6
177#define GD_ZONE_APPEND_USED 7
178
179 struct mutex open_mutex; /* open/close mutex */
180 unsigned open_partitions; /* number of open partitions */
181
182 struct backing_dev_info *bdi;
183 struct kobject queue_kobj; /* the queue/ directory */
184 struct kobject *slave_dir;
185#ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
186 struct list_head slave_bdevs;
187#endif
188 struct timer_rand_state *random;
189 struct disk_events *ev;
190
191#ifdef CONFIG_BLK_DEV_ZONED
192 /*
193 * Zoned block device information. Reads of this information must be
194 * protected with blk_queue_enter() / blk_queue_exit(). Modifying this
195 * information is only allowed while no requests are being processed.
196 * See also blk_mq_freeze_queue() and blk_mq_unfreeze_queue().
197 */
198 unsigned int nr_zones;
199 unsigned int zone_capacity;
200 unsigned int last_zone_capacity;
201 u8 __rcu *zones_cond;
202 unsigned int zone_wplugs_hash_bits;
203 atomic_t nr_zone_wplugs;
204 spinlock_t zone_wplugs_hash_lock;
205 struct mempool *zone_wplugs_pool;
206 struct hlist_head *zone_wplugs_hash;
207 struct workqueue_struct *zone_wplugs_wq;
208 spinlock_t zone_wplugs_list_lock;
209 struct list_head zone_wplugs_list;
210 struct task_struct *zone_wplugs_worker;
211 struct completion zone_wplugs_worker_bio_done;
212#endif /* CONFIG_BLK_DEV_ZONED */
213
214#if IS_ENABLED(CONFIG_CDROM)
215 struct cdrom_device_info *cdi;
216#endif
217 int node_id;
218 struct badblocks *bb;
219 struct lockdep_map lockdep_map;
220 u64 diskseq;
221 blk_mode_t open_mode;
222
223 /*
224 * Independent sector access ranges. This is always NULL for
225 * devices that do not have multiple independent access ranges.
226 */
227 struct blk_independent_access_ranges *ia_ranges;
228
229 struct mutex rqos_state_mutex; /* rqos state change mutex */
230};
231
232/**
233 * disk_openers - returns how many openers are there for a disk
234 * @disk: disk to check
235 *
236 * This returns the number of openers for a disk. Note that this value is only
237 * stable if disk->open_mutex is held.
238 *
239 * Note: Due to a quirk in the block layer open code, each open partition is
240 * only counted once even if there are multiple openers.
241 */
242static inline unsigned int disk_openers(struct gendisk *disk)
243{
244 return atomic_read(&disk->part0->bd_openers);
245}
246
247/**
248 * disk_has_partscan - return %true if partition scanning is enabled on a disk
249 * @disk: disk to check
250 *
251 * Returns %true if partitions scanning is enabled for @disk, or %false if
252 * partition scanning is disabled either permanently or temporarily.
253 */
254static inline bool disk_has_partscan(struct gendisk *disk)
255{
256 return !(disk->flags & (GENHD_FL_NO_PART | GENHD_FL_HIDDEN)) &&
257 !test_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
258}
259
260/*
261 * The gendisk is refcounted by the part0 block_device, and the bd_device
262 * therein is also used for device model presentation in sysfs.
263 */
264#define dev_to_disk(device) \
265 (dev_to_bdev(device)->bd_disk)
266#define disk_to_dev(disk) \
267 (&((disk)->part0->bd_device))
268
269#if IS_REACHABLE(CONFIG_CDROM)
270#define disk_to_cdi(disk) ((disk)->cdi)
271#else
272#define disk_to_cdi(disk) NULL
273#endif
274
275static inline dev_t disk_devt(struct gendisk *disk)
276{
277 return MKDEV(disk->major, disk->first_minor);
278}
279
280#ifdef CONFIG_TRANSPARENT_HUGEPAGE
281/*
282 * We should strive for 1 << (PAGE_SHIFT + MAX_PAGECACHE_ORDER)
283 * however we constrain this to what we can validate and test.
284 */
285#define BLK_MAX_BLOCK_SIZE SZ_64K
286#else
287#define BLK_MAX_BLOCK_SIZE PAGE_SIZE
288#endif
289
290
291/* blk_validate_limits() validates bsize, so drivers don't usually need to */
292static inline int blk_validate_block_size(unsigned long bsize)
293{
294 if (bsize < 512 || bsize > BLK_MAX_BLOCK_SIZE || !is_power_of_2(bsize))
295 return -EINVAL;
296
297 return 0;
298}
299
300static inline bool blk_op_is_passthrough(blk_opf_t op)
301{
302 op &= REQ_OP_MASK;
303 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
304}
305
306/* flags set by the driver in queue_limits.features */
307typedef unsigned int __bitwise blk_features_t;
308
309/* supports a volatile write cache */
310#define BLK_FEAT_WRITE_CACHE ((__force blk_features_t)(1u << 0))
311
312/* supports passing on the FUA bit */
313#define BLK_FEAT_FUA ((__force blk_features_t)(1u << 1))
314
315/* rotational device (hard drive or floppy) */
316#define BLK_FEAT_ROTATIONAL ((__force blk_features_t)(1u << 2))
317
318/* contributes to the random number pool */
319#define BLK_FEAT_ADD_RANDOM ((__force blk_features_t)(1u << 3))
320
321/* do disk/partitions IO accounting */
322#define BLK_FEAT_IO_STAT ((__force blk_features_t)(1u << 4))
323
324/* don't modify data until writeback is done */
325#define BLK_FEAT_STABLE_WRITES ((__force blk_features_t)(1u << 5))
326
327/* always completes in submit context */
328#define BLK_FEAT_SYNCHRONOUS ((__force blk_features_t)(1u << 6))
329
330/* supports REQ_NOWAIT */
331#define BLK_FEAT_NOWAIT ((__force blk_features_t)(1u << 7))
332
333/* supports DAX */
334#define BLK_FEAT_DAX ((__force blk_features_t)(1u << 8))
335
336/* supports I/O polling */
337#define BLK_FEAT_POLL ((__force blk_features_t)(1u << 9))
338
339/* is a zoned device */
340#define BLK_FEAT_ZONED ((__force blk_features_t)(1u << 10))
341
342/* supports PCI(e) p2p requests */
343#define BLK_FEAT_PCI_P2PDMA ((__force blk_features_t)(1u << 12))
344
345/* skip this queue in blk_mq_(un)quiesce_tagset */
346#define BLK_FEAT_SKIP_TAGSET_QUIESCE ((__force blk_features_t)(1u << 13))
347
348/* atomic writes enabled */
349#define BLK_FEAT_ATOMIC_WRITES ((__force blk_features_t)(1u << 14))
350
351/* undocumented magic for bcache */
352#define BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE \
353 ((__force blk_features_t)(1u << 15))
354
355/*
356 * Flags automatically inherited when stacking limits.
357 */
358#define BLK_FEAT_INHERIT_MASK \
359 (BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL | \
360 BLK_FEAT_STABLE_WRITES | BLK_FEAT_ZONED | \
361 BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE)
362
363/* internal flags in queue_limits.flags */
364typedef unsigned int __bitwise blk_flags_t;
365
366/* do not send FLUSH/FUA commands despite advertising a write cache */
367#define BLK_FLAG_WRITE_CACHE_DISABLED ((__force blk_flags_t)(1u << 0))
368
369/* I/O topology is misaligned */
370#define BLK_FLAG_MISALIGNED ((__force blk_flags_t)(1u << 1))
371
372/* passthrough command IO accounting */
373#define BLK_FLAG_IOSTATS_PASSTHROUGH ((__force blk_flags_t)(1u << 2))
374
375struct queue_limits {
376 blk_features_t features;
377 blk_flags_t flags;
378 unsigned long seg_boundary_mask;
379 unsigned long virt_boundary_mask;
380
381 unsigned int max_hw_sectors;
382 unsigned int max_dev_sectors;
383 unsigned int chunk_sectors;
384 unsigned int max_sectors;
385 unsigned int max_user_sectors;
386 unsigned int max_segment_size;
387 unsigned int max_fast_segment_size;
388 unsigned int physical_block_size;
389 unsigned int logical_block_size;
390 unsigned int alignment_offset;
391 unsigned int io_min;
392 unsigned int io_opt;
393 unsigned int max_discard_sectors;
394 unsigned int max_hw_discard_sectors;
395 unsigned int max_user_discard_sectors;
396 unsigned int max_secure_erase_sectors;
397 unsigned int max_write_zeroes_sectors;
398 unsigned int max_wzeroes_unmap_sectors;
399 unsigned int max_hw_wzeroes_unmap_sectors;
400 unsigned int max_user_wzeroes_unmap_sectors;
401 unsigned int max_hw_zone_append_sectors;
402 unsigned int max_zone_append_sectors;
403 unsigned int discard_granularity;
404 unsigned int discard_alignment;
405 unsigned int zone_write_granularity;
406
407 /* atomic write limits */
408 unsigned int atomic_write_hw_max;
409 unsigned int atomic_write_max_sectors;
410 unsigned int atomic_write_hw_boundary;
411 unsigned int atomic_write_boundary_sectors;
412 unsigned int atomic_write_hw_unit_min;
413 unsigned int atomic_write_unit_min;
414 unsigned int atomic_write_hw_unit_max;
415 unsigned int atomic_write_unit_max;
416
417 unsigned short max_segments;
418 unsigned short max_integrity_segments;
419 unsigned short max_discard_segments;
420
421 unsigned short max_write_streams;
422 unsigned int write_stream_granularity;
423
424 unsigned int max_open_zones;
425 unsigned int max_active_zones;
426
427 /*
428 * Drivers that set dma_alignment to less than 511 must be prepared to
429 * handle individual bvec's that are not a multiple of a SECTOR_SIZE
430 * due to possible offsets.
431 */
432 unsigned int dma_alignment;
433 unsigned int dma_pad_mask;
434
435 struct blk_integrity integrity;
436};
437
438typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
439 void *data);
440
441int disk_report_zone(struct gendisk *disk, struct blk_zone *zone,
442 unsigned int idx, struct blk_report_zones_args *args);
443
444int blkdev_get_zone_info(struct block_device *bdev, sector_t sector,
445 struct blk_zone *zone);
446
447#define BLK_ALL_ZONES ((unsigned int)-1)
448int blkdev_report_zones(struct block_device *bdev, sector_t sector,
449 unsigned int nr_zones, report_zones_cb cb, void *data);
450int blkdev_report_zones_cached(struct block_device *bdev, sector_t sector,
451 unsigned int nr_zones, report_zones_cb cb, void *data);
452int blkdev_zone_mgmt(struct block_device *bdev, enum req_op op,
453 sector_t sectors, sector_t nr_sectors);
454int blk_revalidate_disk_zones(struct gendisk *disk);
455
456/*
457 * Independent access ranges: struct blk_independent_access_range describes
458 * a range of contiguous sectors that can be accessed using device command
459 * execution resources that are independent from the resources used for
460 * other access ranges. This is typically found with single-LUN multi-actuator
461 * HDDs where each access range is served by a different set of heads.
462 * The set of independent ranges supported by the device is defined using
463 * struct blk_independent_access_ranges. The independent ranges must not overlap
464 * and must include all sectors within the disk capacity (no sector holes
465 * allowed).
466 * For a device with multiple ranges, requests targeting sectors in different
467 * ranges can be executed in parallel. A request can straddle an access range
468 * boundary.
469 */
470struct blk_independent_access_range {
471 struct kobject kobj;
472 sector_t sector;
473 sector_t nr_sectors;
474};
475
476struct blk_independent_access_ranges {
477 struct kobject kobj;
478 bool sysfs_registered;
479 unsigned int nr_ia_ranges;
480 struct blk_independent_access_range ia_range[];
481};
482
483struct request_queue {
484 /*
485 * The queue owner gets to use this for whatever they like.
486 * ll_rw_blk doesn't touch it.
487 */
488 void *queuedata;
489
490 struct elevator_queue *elevator;
491
492 const struct blk_mq_ops *mq_ops;
493
494 /* sw queues */
495 struct blk_mq_ctx __percpu *queue_ctx;
496
497 /*
498 * various queue flags, see QUEUE_* below
499 */
500 unsigned long queue_flags;
501
502 unsigned int __data_racy rq_timeout;
503
504 unsigned int queue_depth;
505
506 refcount_t refs;
507
508 /* hw dispatch queues */
509 unsigned int nr_hw_queues;
510 struct blk_mq_hw_ctx * __rcu *queue_hw_ctx __counted_by_ptr(nr_hw_queues);
511
512 struct percpu_ref q_usage_counter;
513 struct lock_class_key io_lock_cls_key;
514 struct lockdep_map io_lockdep_map;
515
516 struct lock_class_key q_lock_cls_key;
517 struct lockdep_map q_lockdep_map;
518
519 struct request *last_merge;
520
521 spinlock_t queue_lock;
522
523 int quiesce_depth;
524
525 struct gendisk *disk;
526
527 /*
528 * mq queue kobject
529 */
530 struct kobject *mq_kobj;
531
532 struct queue_limits limits;
533
534#ifdef CONFIG_PM
535 struct device *dev;
536 enum rpm_status rpm_status;
537#endif
538
539 /*
540 * Number of contexts that have called blk_set_pm_only(). If this
541 * counter is above zero then only RQF_PM requests are processed.
542 */
543 atomic_t pm_only;
544
545 struct blk_queue_stats *stats;
546 struct rq_qos *rq_qos;
547 struct mutex rq_qos_mutex;
548
549 /*
550 * ida allocated id for this queue. Used to index queues from
551 * ioctx.
552 */
553 int id;
554
555 /*
556 * queue settings
557 */
558 unsigned int nr_requests; /* Max # of requests */
559 unsigned int async_depth; /* Max # of async requests */
560
561#ifdef CONFIG_BLK_INLINE_ENCRYPTION
562 struct blk_crypto_profile *crypto_profile;
563 struct kobject *crypto_kobject;
564#endif
565
566 struct timer_list timeout;
567 struct work_struct timeout_work;
568
569 atomic_t nr_active_requests_shared_tags;
570
571 struct blk_mq_tags *sched_shared_tags;
572
573 struct list_head icq_list;
574#ifdef CONFIG_BLK_CGROUP
575 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
576 struct blkcg_gq *root_blkg;
577 struct list_head blkg_list;
578 struct mutex blkcg_mutex;
579#endif
580
581 int node;
582
583 spinlock_t requeue_lock;
584 struct list_head requeue_list;
585 struct delayed_work requeue_work;
586
587#ifdef CONFIG_BLK_DEV_IO_TRACE
588 struct blk_trace __rcu *blk_trace;
589#endif
590 /*
591 * for flush operations
592 */
593 struct blk_flush_queue *fq;
594 struct list_head flush_list;
595
596 /*
597 * Protects against I/O scheduler switching, particularly when updating
598 * q->elevator. Since the elevator update code path may also modify q->
599 * nr_requests and wbt latency, this lock also protects the sysfs attrs
600 * nr_requests and wbt_lat_usec. Additionally the nr_hw_queues update
601 * may modify hctx tags, reserved-tags and cpumask, so this lock also
602 * helps protect the hctx sysfs/debugfs attrs. To ensure proper locking
603 * order during an elevator or nr_hw_queue update, first freeze the
604 * queue, then acquire ->elevator_lock.
605 */
606 struct mutex elevator_lock;
607
608 struct mutex sysfs_lock;
609 /*
610 * Protects queue limits and also sysfs attribute read_ahead_kb.
611 */
612 struct mutex limits_lock;
613
614 /*
615 * for reusing dead hctx instance in case of updating
616 * nr_hw_queues
617 */
618 struct list_head unused_hctx_list;
619 spinlock_t unused_hctx_lock;
620
621 int mq_freeze_depth;
622
623#ifdef CONFIG_BLK_DEV_THROTTLING
624 /* Throttle data */
625 struct throtl_data *td;
626#endif
627 struct rcu_head rcu_head;
628#ifdef CONFIG_LOCKDEP
629 struct task_struct *mq_freeze_owner;
630 int mq_freeze_owner_depth;
631 /*
632 * Records disk & queue state in current context, used in unfreeze
633 * queue
634 */
635 bool mq_freeze_disk_dead;
636 bool mq_freeze_queue_dying;
637#endif
638 wait_queue_head_t mq_freeze_wq;
639 /*
640 * Protect concurrent access to q_usage_counter by
641 * percpu_ref_kill() and percpu_ref_reinit().
642 */
643 struct mutex mq_freeze_lock;
644
645 struct blk_mq_tag_set *tag_set;
646 struct list_head tag_set_list;
647
648 struct dentry *debugfs_dir;
649 struct dentry *sched_debugfs_dir;
650 struct dentry *rqos_debugfs_dir;
651 /*
652 * Serializes all debugfs metadata operations using the above dentries.
653 */
654 struct mutex debugfs_mutex;
655};
656
657/* Keep blk_queue_flag_name[] in sync with the definitions below */
658enum {
659 QUEUE_FLAG_DYING, /* queue being torn down */
660 QUEUE_FLAG_NOMERGES, /* disable merge attempts */
661 QUEUE_FLAG_SAME_COMP, /* complete on same CPU-group */
662 QUEUE_FLAG_FAIL_IO, /* fake timeout */
663 QUEUE_FLAG_NOXMERGES, /* No extended merges */
664 QUEUE_FLAG_SAME_FORCE, /* force complete on same CPU */
665 QUEUE_FLAG_INIT_DONE, /* queue is initialized */
666 QUEUE_FLAG_STATS, /* track IO start and completion times */
667 QUEUE_FLAG_REGISTERED, /* queue has been registered to a disk */
668 QUEUE_FLAG_QUIESCED, /* queue has been quiesced */
669 QUEUE_FLAG_RQ_ALLOC_TIME, /* record rq->alloc_time_ns */
670 QUEUE_FLAG_HCTX_ACTIVE, /* at least one blk-mq hctx is active */
671 QUEUE_FLAG_SQ_SCHED, /* single queue style io dispatch */
672 QUEUE_FLAG_DISABLE_WBT_DEF, /* for sched to disable/enable wbt */
673 QUEUE_FLAG_NO_ELV_SWITCH, /* can't switch elevator any more */
674 QUEUE_FLAG_QOS_ENABLED, /* qos is enabled */
675 QUEUE_FLAG_BIO_ISSUE_TIME, /* record bio->issue_time_ns */
676 QUEUE_FLAG_ZONED_QD1_WRITES, /* Limit zoned devices writes to QD=1 */
677 QUEUE_FLAG_MAX
678};
679
680#define QUEUE_FLAG_MQ_DEFAULT (1UL << QUEUE_FLAG_SAME_COMP)
681
682void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
683void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
684
685#define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
686#define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
687#define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
688#define blk_queue_noxmerges(q) \
689 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
690#define blk_queue_rot(q) ((q)->limits.features & BLK_FEAT_ROTATIONAL)
691#define blk_queue_io_stat(q) ((q)->limits.features & BLK_FEAT_IO_STAT)
692#define blk_queue_passthrough_stat(q) \
693 ((q)->limits.flags & BLK_FLAG_IOSTATS_PASSTHROUGH)
694#define blk_queue_dax(q) ((q)->limits.features & BLK_FEAT_DAX)
695#define blk_queue_pci_p2pdma(q) ((q)->limits.features & BLK_FEAT_PCI_P2PDMA)
696#ifdef CONFIG_BLK_RQ_ALLOC_TIME
697#define blk_queue_rq_alloc_time(q) \
698 test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
699#else
700#define blk_queue_rq_alloc_time(q) false
701#endif
702
703#define blk_noretry_request(rq) \
704 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
705 REQ_FAILFAST_DRIVER))
706#define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
707#define blk_queue_pm_only(q) atomic_read(&(q)->pm_only)
708#define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
709#define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags)
710#define blk_queue_skip_tagset_quiesce(q) \
711 ((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE)
712#define blk_queue_disable_wbt(q) \
713 test_bit(QUEUE_FLAG_DISABLE_WBT_DEF, &(q)->queue_flags)
714#define blk_queue_no_elv_switch(q) \
715 test_bit(QUEUE_FLAG_NO_ELV_SWITCH, &(q)->queue_flags)
716#define blk_queue_zoned_qd1_writes(q) \
717 test_bit(QUEUE_FLAG_ZONED_QD1_WRITES, &(q)->queue_flags)
718
719extern void blk_set_pm_only(struct request_queue *q);
720extern void blk_clear_pm_only(struct request_queue *q);
721
722#define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
723
724#define dma_map_bvec(dev, bv, dir, attrs) \
725 dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \
726 (dir), (attrs))
727
728static inline bool queue_is_mq(struct request_queue *q)
729{
730 return q->mq_ops;
731}
732
733#ifdef CONFIG_PM
734static inline enum rpm_status queue_rpm_status(struct request_queue *q)
735{
736 return q->rpm_status;
737}
738#else
739static inline enum rpm_status queue_rpm_status(struct request_queue *q)
740{
741 return RPM_ACTIVE;
742}
743#endif
744
745static inline bool blk_queue_is_zoned(struct request_queue *q)
746{
747 return IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
748 (q->limits.features & BLK_FEAT_ZONED);
749}
750
751static inline unsigned int disk_zone_no(struct gendisk *disk, sector_t sector)
752{
753 if (!blk_queue_is_zoned(disk->queue))
754 return 0;
755 return sector >> ilog2(disk->queue->limits.chunk_sectors);
756}
757
758static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
759{
760 return bdev->bd_disk->queue->limits.max_open_zones;
761}
762
763static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
764{
765 return bdev->bd_disk->queue->limits.max_active_zones;
766}
767
768static inline unsigned int blk_queue_depth(struct request_queue *q)
769{
770 if (q->queue_depth)
771 return q->queue_depth;
772
773 return q->nr_requests;
774}
775
776/*
777 * default timeout for SG_IO if none specified
778 */
779#define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
780#define BLK_MIN_SG_TIMEOUT (7 * HZ)
781
782/* This should not be used directly - use rq_for_each_segment */
783#define for_each_bio(_bio) \
784 for (; _bio; _bio = _bio->bi_next)
785
786int __must_check add_disk_fwnode(struct device *parent, struct gendisk *disk,
787 const struct attribute_group **groups,
788 struct fwnode_handle *fwnode);
789int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
790 const struct attribute_group **groups);
791static inline int __must_check add_disk(struct gendisk *disk)
792{
793 return device_add_disk(NULL, disk, NULL);
794}
795void del_gendisk(struct gendisk *gp);
796void invalidate_disk(struct gendisk *disk);
797void set_disk_ro(struct gendisk *disk, bool read_only);
798void disk_uevent(struct gendisk *disk, enum kobject_action action);
799
800static inline u8 bdev_partno(const struct block_device *bdev)
801{
802 return atomic_read(&bdev->__bd_flags) & BD_PARTNO;
803}
804
805static inline bool bdev_test_flag(const struct block_device *bdev, unsigned flag)
806{
807 return atomic_read(&bdev->__bd_flags) & flag;
808}
809
810static inline void bdev_set_flag(struct block_device *bdev, unsigned flag)
811{
812 atomic_or(flag, &bdev->__bd_flags);
813}
814
815static inline void bdev_clear_flag(struct block_device *bdev, unsigned flag)
816{
817 atomic_andnot(flag, &bdev->__bd_flags);
818}
819
820static inline bool get_disk_ro(struct gendisk *disk)
821{
822 return bdev_test_flag(disk->part0, BD_READ_ONLY) ||
823 test_bit(GD_READ_ONLY, &disk->state);
824}
825
826static inline bool bdev_read_only(struct block_device *bdev)
827{
828 return bdev_test_flag(bdev, BD_READ_ONLY) || get_disk_ro(bdev->bd_disk);
829}
830
831bool set_capacity_and_notify(struct gendisk *disk, sector_t size);
832void disk_force_media_change(struct gendisk *disk);
833void bdev_mark_dead(struct block_device *bdev, bool surprise);
834
835void add_disk_randomness(struct gendisk *disk) __latent_entropy;
836void rand_initialize_disk(struct gendisk *disk);
837
838static inline sector_t get_start_sect(struct block_device *bdev)
839{
840 return bdev->bd_start_sect;
841}
842
843static inline sector_t bdev_nr_sectors(struct block_device *bdev)
844{
845 return bdev->bd_nr_sectors;
846}
847
848static inline loff_t bdev_nr_bytes(struct block_device *bdev)
849{
850 return (loff_t)bdev_nr_sectors(bdev) << SECTOR_SHIFT;
851}
852
853static inline sector_t get_capacity(struct gendisk *disk)
854{
855 return bdev_nr_sectors(disk->part0);
856}
857
858static inline u64 sb_bdev_nr_blocks(struct super_block *sb)
859{
860 return bdev_nr_sectors(sb->s_bdev) >>
861 (sb->s_blocksize_bits - SECTOR_SHIFT);
862}
863
864#ifdef CONFIG_BLK_DEV_ZONED
865static inline unsigned int disk_nr_zones(struct gendisk *disk)
866{
867 return disk->nr_zones;
868}
869
870/**
871 * bio_needs_zone_write_plugging - Check if a BIO needs to be handled with zone
872 * write plugging
873 * @bio: The BIO being submitted
874 *
875 * Return true whenever @bio execution needs to be handled through zone
876 * write plugging (using blk_zone_plug_bio()). Return false otherwise.
877 */
878static inline bool bio_needs_zone_write_plugging(struct bio *bio)
879{
880 enum req_op op = bio_op(bio);
881
882 /*
883 * Only zoned block devices have a zone write plug hash table. But not
884 * all of them have one (e.g. DM devices may not need one).
885 */
886 if (!bio->bi_bdev->bd_disk->zone_wplugs_hash)
887 return false;
888
889 /* Only write operations need zone write plugging. */
890 if (!op_is_write(op))
891 return false;
892
893 /* Ignore empty flush */
894 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
895 return false;
896
897 /* Ignore BIOs that already have been handled by zone write plugging. */
898 if (bio_flagged(bio, BIO_ZONE_WRITE_PLUGGING))
899 return false;
900
901 /*
902 * All zone write operations must be handled through zone write plugging
903 * using blk_zone_plug_bio().
904 */
905 switch (op) {
906 case REQ_OP_ZONE_APPEND:
907 case REQ_OP_WRITE:
908 case REQ_OP_WRITE_ZEROES:
909 case REQ_OP_ZONE_FINISH:
910 case REQ_OP_ZONE_RESET:
911 case REQ_OP_ZONE_RESET_ALL:
912 return true;
913 default:
914 return false;
915 }
916}
917
918bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs);
919
920/**
921 * disk_zone_capacity - returns the zone capacity of zone containing @sector
922 * @disk: disk to work with
923 * @sector: sector number within the querying zone
924 *
925 * Returns the zone capacity of a zone containing @sector. @sector can be any
926 * sector in the zone.
927 */
928static inline unsigned int disk_zone_capacity(struct gendisk *disk,
929 sector_t sector)
930{
931 sector_t zone_sectors = disk->queue->limits.chunk_sectors;
932
933 if (sector + zone_sectors >= get_capacity(disk))
934 return disk->last_zone_capacity;
935 return disk->zone_capacity;
936}
937static inline unsigned int bdev_zone_capacity(struct block_device *bdev,
938 sector_t pos)
939{
940 return disk_zone_capacity(bdev->bd_disk, pos);
941}
942
943bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector);
944
945#else /* CONFIG_BLK_DEV_ZONED */
946static inline unsigned int disk_nr_zones(struct gendisk *disk)
947{
948 return 0;
949}
950
951static inline bool bdev_zone_is_seq(struct block_device *bdev, sector_t sector)
952{
953 return false;
954}
955
956static inline bool bio_needs_zone_write_plugging(struct bio *bio)
957{
958 return false;
959}
960
961static inline bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs)
962{
963 return false;
964}
965#endif /* CONFIG_BLK_DEV_ZONED */
966
967static inline unsigned int bdev_nr_zones(struct block_device *bdev)
968{
969 return disk_nr_zones(bdev->bd_disk);
970}
971
972int bdev_disk_changed(struct gendisk *disk, bool invalidate);
973
974void put_disk(struct gendisk *disk);
975struct gendisk *__blk_alloc_disk(struct queue_limits *lim, int node,
976 struct lock_class_key *lkclass);
977
978/**
979 * blk_alloc_disk - allocate a gendisk structure
980 * @lim: queue limits to be used for this disk.
981 * @node_id: numa node to allocate on
982 *
983 * Allocate and pre-initialize a gendisk structure for use with BIO based
984 * drivers.
985 *
986 * Returns an ERR_PTR on error, else the allocated disk.
987 *
988 * Context: can sleep
989 */
990#define blk_alloc_disk(lim, node_id) \
991({ \
992 static struct lock_class_key __key; \
993 \
994 __blk_alloc_disk(lim, node_id, &__key); \
995})
996
997int __register_blkdev(unsigned int major, const char *name,
998 void (*probe)(dev_t devt));
999#define register_blkdev(major, name) \
1000 __register_blkdev(major, name, NULL)
1001void unregister_blkdev(unsigned int major, const char *name);
1002
1003bool disk_check_media_change(struct gendisk *disk);
1004void set_capacity(struct gendisk *disk, sector_t size);
1005
1006#ifdef CONFIG_BLOCK_HOLDER_DEPRECATED
1007int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk);
1008void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk);
1009#else
1010static inline int bd_link_disk_holder(struct block_device *bdev,
1011 struct gendisk *disk)
1012{
1013 return 0;
1014}
1015static inline void bd_unlink_disk_holder(struct block_device *bdev,
1016 struct gendisk *disk)
1017{
1018}
1019#endif /* CONFIG_BLOCK_HOLDER_DEPRECATED */
1020
1021dev_t part_devt(struct gendisk *disk, u8 partno);
1022void inc_diskseq(struct gendisk *disk);
1023void blk_request_module(dev_t devt);
1024
1025extern int blk_register_queue(struct gendisk *disk);
1026extern void blk_unregister_queue(struct gendisk *disk);
1027void submit_bio_noacct(struct bio *bio);
1028struct bio *bio_split_to_limits(struct bio *bio);
1029struct bio *bio_submit_split_bioset(struct bio *bio, unsigned int split_sectors,
1030 struct bio_set *bs);
1031
1032extern int blk_lld_busy(struct request_queue *q);
1033extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
1034extern void blk_queue_exit(struct request_queue *q);
1035extern void blk_sync_queue(struct request_queue *q);
1036
1037/* Convert a request operation REQ_OP_name into the string "name" */
1038extern const char *blk_op_str(enum req_op op);
1039
1040int blk_status_to_errno(blk_status_t status);
1041blk_status_t errno_to_blk_status(int errno);
1042const char *blk_status_to_str(blk_status_t status);
1043
1044/* only poll the hardware once, don't continue until a completion was found */
1045#define BLK_POLL_ONESHOT (1 << 0)
1046int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags);
1047int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
1048 unsigned int flags);
1049
1050static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
1051{
1052 return bdev->bd_queue; /* this is never NULL */
1053}
1054
1055/* Convert a zone condition BLK_ZONE_COND_name into the string "name" */
1056const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
1057
1058static inline unsigned int bio_zone_no(struct bio *bio)
1059{
1060 return disk_zone_no(bio->bi_bdev->bd_disk, bio->bi_iter.bi_sector);
1061}
1062
1063static inline bool bio_straddles_zones(struct bio *bio)
1064{
1065 return bio_sectors(bio) &&
1066 bio_zone_no(bio) !=
1067 disk_zone_no(bio->bi_bdev->bd_disk, bio_end_sector(bio) - 1);
1068}
1069
1070/*
1071 * Return how much within the boundary is left to be used for I/O at a given
1072 * offset.
1073 */
1074static inline unsigned int blk_boundary_sectors_left(sector_t offset,
1075 unsigned int boundary_sectors)
1076{
1077 if (unlikely(!is_power_of_2(boundary_sectors)))
1078 return boundary_sectors - sector_div(offset, boundary_sectors);
1079 return boundary_sectors - (offset & (boundary_sectors - 1));
1080}
1081
1082/**
1083 * queue_limits_start_update - start an atomic update of queue limits
1084 * @q: queue to update
1085 *
1086 * This functions starts an atomic update of the queue limits. It takes a lock
1087 * to prevent other updates and returns a snapshot of the current limits that
1088 * the caller can modify. The caller must call queue_limits_commit_update()
1089 * to finish the update.
1090 *
1091 * Context: process context.
1092 */
1093static inline struct queue_limits
1094queue_limits_start_update(struct request_queue *q)
1095{
1096 mutex_lock(&q->limits_lock);
1097 return q->limits;
1098}
1099int queue_limits_commit_update_frozen(struct request_queue *q,
1100 struct queue_limits *lim);
1101int queue_limits_commit_update(struct request_queue *q,
1102 struct queue_limits *lim);
1103int queue_limits_set(struct request_queue *q, struct queue_limits *lim);
1104int blk_validate_limits(struct queue_limits *lim);
1105
1106/**
1107 * queue_limits_cancel_update - cancel an atomic update of queue limits
1108 * @q: queue to update
1109 *
1110 * This functions cancels an atomic update of the queue limits started by
1111 * queue_limits_start_update() and should be used when an error occurs after
1112 * starting update.
1113 */
1114static inline void queue_limits_cancel_update(struct request_queue *q)
1115{
1116 mutex_unlock(&q->limits_lock);
1117}
1118
1119/*
1120 * These helpers are for drivers that have sloppy feature negotiation and might
1121 * have to disable DISCARD, WRITE_ZEROES or SECURE_DISCARD from the I/O
1122 * completion handler when the device returned an indicator that the respective
1123 * feature is not actually supported. They are racy and the driver needs to
1124 * cope with that. Try to avoid this scheme if you can.
1125 */
1126static inline void blk_queue_disable_discard(struct request_queue *q)
1127{
1128 q->limits.max_discard_sectors = 0;
1129}
1130
1131static inline void blk_queue_disable_secure_erase(struct request_queue *q)
1132{
1133 q->limits.max_secure_erase_sectors = 0;
1134}
1135
1136static inline void blk_queue_disable_write_zeroes(struct request_queue *q)
1137{
1138 q->limits.max_write_zeroes_sectors = 0;
1139 q->limits.max_wzeroes_unmap_sectors = 0;
1140}
1141
1142/*
1143 * Access functions for manipulating queue properties
1144 */
1145extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
1146extern void blk_set_stacking_limits(struct queue_limits *lim);
1147extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
1148 sector_t offset);
1149void queue_limits_stack_bdev(struct queue_limits *t, struct block_device *bdev,
1150 sector_t offset, const char *pfx);
1151extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1152
1153struct blk_independent_access_ranges *
1154disk_alloc_independent_access_ranges(struct gendisk *disk, int nr_ia_ranges);
1155void disk_set_independent_access_ranges(struct gendisk *disk,
1156 struct blk_independent_access_ranges *iars);
1157
1158bool __must_check blk_get_queue(struct request_queue *);
1159extern void blk_put_queue(struct request_queue *);
1160
1161void blk_mark_disk_dead(struct gendisk *disk);
1162
1163struct rq_list {
1164 struct request *head;
1165 struct request *tail;
1166};
1167
1168#ifdef CONFIG_BLOCK
1169/*
1170 * blk_plug permits building a queue of related requests by holding the I/O
1171 * fragments for a short period. This allows merging of sequential requests
1172 * into single larger request. As the requests are moved from a per-task list to
1173 * the device's request_queue in a batch, this results in improved scalability
1174 * as the lock contention for request_queue lock is reduced.
1175 *
1176 * It is ok not to disable preemption when adding the request to the plug list
1177 * or when attempting a merge. For details, please see schedule() where
1178 * blk_flush_plug() is called.
1179 */
1180struct blk_plug {
1181 struct rq_list mq_list; /* blk-mq requests */
1182
1183 /* if ios_left is > 1, we can batch tag/rq allocations */
1184 struct rq_list cached_rqs;
1185 u64 cur_ktime;
1186 unsigned short nr_ios;
1187
1188 unsigned short rq_count;
1189
1190 bool multiple_queues;
1191 bool has_elevator;
1192
1193 struct list_head cb_list; /* md requires an unplug callback */
1194};
1195
1196struct blk_plug_cb;
1197typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1198struct blk_plug_cb {
1199 struct list_head list;
1200 blk_plug_cb_fn callback;
1201 void *data;
1202};
1203extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
1204 void *data, int size);
1205extern void blk_start_plug(struct blk_plug *);
1206extern void blk_start_plug_nr_ios(struct blk_plug *, unsigned short);
1207extern void blk_finish_plug(struct blk_plug *);
1208
1209void __blk_flush_plug(struct blk_plug *plug, bool from_schedule);
1210static inline void blk_flush_plug(struct blk_plug *plug, bool async)
1211{
1212 if (plug)
1213 __blk_flush_plug(plug, async);
1214}
1215
1216/*
1217 * tsk == current here
1218 */
1219static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
1220{
1221 struct blk_plug *plug = tsk->plug;
1222
1223 if (plug)
1224 plug->cur_ktime = 0;
1225 current->flags &= ~PF_BLOCK_TS;
1226}
1227
1228int blkdev_issue_flush(struct block_device *bdev);
1229long nr_blockdev_pages(void);
1230#else /* CONFIG_BLOCK */
1231struct blk_plug {
1232};
1233
1234static inline void blk_start_plug_nr_ios(struct blk_plug *plug,
1235 unsigned short nr_ios)
1236{
1237}
1238
1239static inline void blk_start_plug(struct blk_plug *plug)
1240{
1241}
1242
1243static inline void blk_finish_plug(struct blk_plug *plug)
1244{
1245}
1246
1247static inline void blk_flush_plug(struct blk_plug *plug, bool async)
1248{
1249}
1250
1251static inline void blk_plug_invalidate_ts(struct task_struct *tsk)
1252{
1253}
1254
1255static inline int blkdev_issue_flush(struct block_device *bdev)
1256{
1257 return 0;
1258}
1259
1260static inline long nr_blockdev_pages(void)
1261{
1262 return 0;
1263}
1264#endif /* CONFIG_BLOCK */
1265
1266extern void blk_io_schedule(void);
1267
1268int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1269 sector_t nr_sects, gfp_t gfp_mask);
1270void __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1271 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop);
1272int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
1273 sector_t nr_sects, gfp_t gfp);
1274
1275#define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */
1276#define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */
1277#define BLKDEV_ZERO_KILLABLE (1 << 2) /* interruptible by fatal signals */
1278
1279extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1280 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1281 unsigned flags);
1282extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1283 sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
1284
1285static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1286 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1287{
1288 return blkdev_issue_discard(sb->s_bdev,
1289 block << (sb->s_blocksize_bits -
1290 SECTOR_SHIFT),
1291 nr_blocks << (sb->s_blocksize_bits -
1292 SECTOR_SHIFT),
1293 gfp_mask);
1294}
1295static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1296 sector_t nr_blocks, gfp_t gfp_mask)
1297{
1298 return blkdev_issue_zeroout(sb->s_bdev,
1299 block << (sb->s_blocksize_bits -
1300 SECTOR_SHIFT),
1301 nr_blocks << (sb->s_blocksize_bits -
1302 SECTOR_SHIFT),
1303 gfp_mask, 0);
1304}
1305
1306static inline bool bdev_is_partition(struct block_device *bdev)
1307{
1308 return bdev_partno(bdev) != 0;
1309}
1310
1311enum blk_default_limits {
1312 BLK_MAX_SEGMENTS = 128,
1313 BLK_SAFE_MAX_SECTORS = 255,
1314 BLK_MAX_SEGMENT_SIZE = 65536,
1315 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
1316};
1317
1318static inline struct queue_limits *bdev_limits(struct block_device *bdev)
1319{
1320 return &bdev_get_queue(bdev)->limits;
1321}
1322
1323static inline unsigned long queue_segment_boundary(const struct request_queue *q)
1324{
1325 return q->limits.seg_boundary_mask;
1326}
1327
1328static inline unsigned long queue_virt_boundary(const struct request_queue *q)
1329{
1330 return q->limits.virt_boundary_mask;
1331}
1332
1333static inline unsigned int queue_max_sectors(const struct request_queue *q)
1334{
1335 return q->limits.max_sectors;
1336}
1337
1338static inline unsigned int queue_max_bytes(struct request_queue *q)
1339{
1340 return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9;
1341}
1342
1343static inline unsigned int queue_max_hw_sectors(const struct request_queue *q)
1344{
1345 return q->limits.max_hw_sectors;
1346}
1347
1348static inline unsigned short queue_max_segments(const struct request_queue *q)
1349{
1350 return q->limits.max_segments;
1351}
1352
1353static inline unsigned short queue_max_discard_segments(const struct request_queue *q)
1354{
1355 return q->limits.max_discard_segments;
1356}
1357
1358static inline unsigned int queue_max_segment_size(const struct request_queue *q)
1359{
1360 return q->limits.max_segment_size;
1361}
1362
1363static inline bool queue_emulates_zone_append(struct request_queue *q)
1364{
1365 return blk_queue_is_zoned(q) && !q->limits.max_hw_zone_append_sectors;
1366}
1367
1368static inline bool bdev_emulates_zone_append(struct block_device *bdev)
1369{
1370 return queue_emulates_zone_append(bdev_get_queue(bdev));
1371}
1372
1373static inline unsigned int
1374bdev_max_zone_append_sectors(struct block_device *bdev)
1375{
1376 return bdev_limits(bdev)->max_zone_append_sectors;
1377}
1378
1379static inline unsigned int bdev_max_segments(struct block_device *bdev)
1380{
1381 return queue_max_segments(bdev_get_queue(bdev));
1382}
1383
1384static inline unsigned short bdev_max_write_streams(struct block_device *bdev)
1385{
1386 if (bdev_is_partition(bdev))
1387 return 0;
1388 return bdev_limits(bdev)->max_write_streams;
1389}
1390
1391static inline unsigned queue_logical_block_size(const struct request_queue *q)
1392{
1393 return q->limits.logical_block_size;
1394}
1395
1396static inline unsigned int bdev_logical_block_size(struct block_device *bdev)
1397{
1398 return queue_logical_block_size(bdev_get_queue(bdev));
1399}
1400
1401static inline unsigned int queue_physical_block_size(const struct request_queue *q)
1402{
1403 return q->limits.physical_block_size;
1404}
1405
1406static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1407{
1408 return queue_physical_block_size(bdev_get_queue(bdev));
1409}
1410
1411static inline unsigned int queue_io_min(const struct request_queue *q)
1412{
1413 return q->limits.io_min;
1414}
1415
1416static inline unsigned int bdev_io_min(struct block_device *bdev)
1417{
1418 return queue_io_min(bdev_get_queue(bdev));
1419}
1420
1421static inline unsigned int queue_io_opt(const struct request_queue *q)
1422{
1423 return q->limits.io_opt;
1424}
1425
1426static inline unsigned int bdev_io_opt(struct block_device *bdev)
1427{
1428 return queue_io_opt(bdev_get_queue(bdev));
1429}
1430
1431static inline unsigned int
1432queue_zone_write_granularity(const struct request_queue *q)
1433{
1434 return q->limits.zone_write_granularity;
1435}
1436
1437static inline unsigned int
1438bdev_zone_write_granularity(struct block_device *bdev)
1439{
1440 return queue_zone_write_granularity(bdev_get_queue(bdev));
1441}
1442
1443int bdev_alignment_offset(struct block_device *bdev);
1444unsigned int bdev_discard_alignment(struct block_device *bdev);
1445
1446static inline unsigned int bdev_max_discard_sectors(struct block_device *bdev)
1447{
1448 return bdev_limits(bdev)->max_discard_sectors;
1449}
1450
1451static inline unsigned int bdev_discard_granularity(struct block_device *bdev)
1452{
1453 return bdev_limits(bdev)->discard_granularity;
1454}
1455
1456static inline unsigned int
1457bdev_max_secure_erase_sectors(struct block_device *bdev)
1458{
1459 return bdev_limits(bdev)->max_secure_erase_sectors;
1460}
1461
1462static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
1463{
1464 return bdev_limits(bdev)->max_write_zeroes_sectors;
1465}
1466
1467static inline unsigned int
1468bdev_write_zeroes_unmap_sectors(struct block_device *bdev)
1469{
1470 return bdev_limits(bdev)->max_wzeroes_unmap_sectors;
1471}
1472
1473static inline bool bdev_rot(struct block_device *bdev)
1474{
1475 return blk_queue_rot(bdev_get_queue(bdev));
1476}
1477
1478static inline bool bdev_synchronous(struct block_device *bdev)
1479{
1480 return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS;
1481}
1482
1483static inline bool bdev_has_integrity_csum(struct block_device *bdev)
1484{
1485 struct queue_limits *lim = bdev_limits(bdev);
1486
1487 return IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1488 lim->integrity.csum_type != BLK_INTEGRITY_CSUM_NONE;
1489}
1490
1491static inline bool bdev_stable_writes(struct block_device *bdev)
1492{
1493 return bdev_has_integrity_csum(bdev) ||
1494 (bdev_limits(bdev)->features & BLK_FEAT_STABLE_WRITES);
1495}
1496
1497static inline bool blk_queue_write_cache(struct request_queue *q)
1498{
1499 return (q->limits.features & BLK_FEAT_WRITE_CACHE) &&
1500 !(q->limits.flags & BLK_FLAG_WRITE_CACHE_DISABLED);
1501}
1502
1503static inline bool bdev_write_cache(struct block_device *bdev)
1504{
1505 return blk_queue_write_cache(bdev_get_queue(bdev));
1506}
1507
1508static inline bool bdev_fua(struct block_device *bdev)
1509{
1510 return bdev_limits(bdev)->features & BLK_FEAT_FUA;
1511}
1512
1513static inline bool bdev_nowait(struct block_device *bdev)
1514{
1515 return bdev->bd_disk->queue->limits.features & BLK_FEAT_NOWAIT;
1516}
1517
1518static inline bool bdev_is_zoned(struct block_device *bdev)
1519{
1520 return blk_queue_is_zoned(bdev_get_queue(bdev));
1521}
1522
1523static inline unsigned int bdev_zone_no(struct block_device *bdev, sector_t sec)
1524{
1525 return disk_zone_no(bdev->bd_disk, sec);
1526}
1527
1528static inline sector_t bdev_zone_sectors(struct block_device *bdev)
1529{
1530 struct request_queue *q = bdev_get_queue(bdev);
1531
1532 if (!blk_queue_is_zoned(q))
1533 return 0;
1534 return q->limits.chunk_sectors;
1535}
1536
1537static inline sector_t bdev_zone_start(struct block_device *bdev,
1538 sector_t sector)
1539{
1540 return sector & ~(bdev_zone_sectors(bdev) - 1);
1541}
1542
1543static inline sector_t bdev_offset_from_zone_start(struct block_device *bdev,
1544 sector_t sector)
1545{
1546 return sector & (bdev_zone_sectors(bdev) - 1);
1547}
1548
1549static inline sector_t bio_offset_from_zone_start(struct bio *bio)
1550{
1551 return bdev_offset_from_zone_start(bio->bi_bdev,
1552 bio->bi_iter.bi_sector);
1553}
1554
1555static inline bool bdev_is_zone_start(struct block_device *bdev,
1556 sector_t sector)
1557{
1558 return bdev_offset_from_zone_start(bdev, sector) == 0;
1559}
1560
1561/* Check whether @sector is a multiple of the zone size. */
1562static inline bool bdev_is_zone_aligned(struct block_device *bdev,
1563 sector_t sector)
1564{
1565 return bdev_is_zone_start(bdev, sector);
1566}
1567
1568int blk_zone_issue_zeroout(struct block_device *bdev, sector_t sector,
1569 sector_t nr_sects, gfp_t gfp_mask);
1570
1571static inline unsigned int queue_dma_alignment(const struct request_queue *q)
1572{
1573 return q->limits.dma_alignment;
1574}
1575
1576static inline unsigned int
1577queue_atomic_write_unit_max_bytes(const struct request_queue *q)
1578{
1579 return q->limits.atomic_write_unit_max;
1580}
1581
1582static inline unsigned int
1583queue_atomic_write_unit_min_bytes(const struct request_queue *q)
1584{
1585 return q->limits.atomic_write_unit_min;
1586}
1587
1588static inline unsigned int
1589queue_atomic_write_boundary_bytes(const struct request_queue *q)
1590{
1591 return q->limits.atomic_write_boundary_sectors << SECTOR_SHIFT;
1592}
1593
1594static inline unsigned int
1595queue_atomic_write_max_bytes(const struct request_queue *q)
1596{
1597 return q->limits.atomic_write_max_sectors << SECTOR_SHIFT;
1598}
1599
1600static inline unsigned int bdev_dma_alignment(struct block_device *bdev)
1601{
1602 return queue_dma_alignment(bdev_get_queue(bdev));
1603}
1604
1605static inline unsigned int
1606blk_lim_dma_alignment_and_pad(struct queue_limits *lim)
1607{
1608 return lim->dma_alignment | lim->dma_pad_mask;
1609}
1610
1611static inline bool blk_rq_aligned(struct request_queue *q, unsigned long addr,
1612 unsigned int len)
1613{
1614 unsigned int alignment = blk_lim_dma_alignment_and_pad(&q->limits);
1615
1616 return !(addr & alignment) && !(len & alignment);
1617}
1618
1619/* assumes size > 256 */
1620static inline unsigned int blksize_bits(unsigned int size)
1621{
1622 return order_base_2(size >> SECTOR_SHIFT) + SECTOR_SHIFT;
1623}
1624
1625int kblockd_schedule_work(struct work_struct *work);
1626int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1627
1628#define MODULE_ALIAS_BLOCKDEV(major,minor) \
1629 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1630#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1631 MODULE_ALIAS("block-major-" __stringify(major) "-*")
1632
1633#ifdef CONFIG_BLK_INLINE_ENCRYPTION
1634
1635bool blk_crypto_register(struct blk_crypto_profile *profile,
1636 struct request_queue *q);
1637
1638#else /* CONFIG_BLK_INLINE_ENCRYPTION */
1639
1640static inline bool blk_crypto_register(struct blk_crypto_profile *profile,
1641 struct request_queue *q)
1642{
1643 return true;
1644}
1645
1646#endif /* CONFIG_BLK_INLINE_ENCRYPTION */
1647
1648enum blk_unique_id {
1649 /* these match the Designator Types specified in SPC */
1650 BLK_UID_T10 = 1,
1651 BLK_UID_EUI64 = 2,
1652 BLK_UID_NAA = 3,
1653};
1654
1655struct block_device_operations {
1656 void (*submit_bio)(struct bio *bio);
1657 int (*poll_bio)(struct bio *bio, struct io_comp_batch *iob,
1658 unsigned int flags);
1659 int (*open)(struct gendisk *disk, blk_mode_t mode);
1660 void (*release)(struct gendisk *disk);
1661 int (*ioctl)(struct block_device *bdev, blk_mode_t mode,
1662 unsigned cmd, unsigned long arg);
1663 int (*compat_ioctl)(struct block_device *bdev, blk_mode_t mode,
1664 unsigned cmd, unsigned long arg);
1665 unsigned int (*check_events) (struct gendisk *disk,
1666 unsigned int clearing);
1667 void (*unlock_native_capacity) (struct gendisk *);
1668 int (*getgeo)(struct gendisk *, struct hd_geometry *);
1669 int (*set_read_only)(struct block_device *bdev, bool ro);
1670 void (*free_disk)(struct gendisk *disk);
1671 /* this callback is with swap_lock and sometimes page table lock held */
1672 void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1673 int (*report_zones)(struct gendisk *, sector_t sector,
1674 unsigned int nr_zones,
1675 struct blk_report_zones_args *args);
1676 char *(*devnode)(struct gendisk *disk, umode_t *mode);
1677 /* returns the length of the identifier or a negative errno: */
1678 int (*get_unique_id)(struct gendisk *disk, u8 id[16],
1679 enum blk_unique_id id_type);
1680 struct module *owner;
1681 const struct pr_ops *pr_ops;
1682
1683 /*
1684 * Special callback for probing GPT entry at a given sector.
1685 * Needed by Android devices, used by GPT scanner and MMC blk
1686 * driver.
1687 */
1688 int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector);
1689};
1690
1691#ifdef CONFIG_COMPAT
1692extern int blkdev_compat_ptr_ioctl(struct block_device *, blk_mode_t,
1693 unsigned int, unsigned long);
1694#else
1695#define blkdev_compat_ptr_ioctl NULL
1696#endif
1697
1698static inline void blk_wake_io_task(struct task_struct *waiter)
1699{
1700 /*
1701 * If we're polling, the task itself is doing the completions. For
1702 * that case, we don't need to signal a wakeup, it's enough to just
1703 * mark us as RUNNING.
1704 */
1705 if (waiter == current)
1706 __set_current_state(TASK_RUNNING);
1707 else
1708 wake_up_process(waiter);
1709}
1710
1711unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op,
1712 unsigned long start_time);
1713void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
1714 unsigned int sectors, unsigned long start_time);
1715
1716unsigned long bio_start_io_acct(struct bio *bio);
1717void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1718 struct block_device *orig_bdev);
1719
1720/**
1721 * bio_end_io_acct - end I/O accounting for bio based drivers
1722 * @bio: bio to end account for
1723 * @start_time: start time returned by bio_start_io_acct()
1724 */
1725static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
1726{
1727 return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev);
1728}
1729
1730int bdev_validate_blocksize(struct block_device *bdev, int block_size);
1731int set_blocksize(struct file *file, int size);
1732
1733int lookup_bdev(const char *pathname, dev_t *dev);
1734
1735void blkdev_show(struct seq_file *seqf, off_t offset);
1736
1737#define BDEVNAME_SIZE 32 /* Largest string for a blockdev identifier */
1738#define BDEVT_SIZE 10 /* Largest string for MAJ:MIN for blkdev */
1739#ifdef CONFIG_BLOCK
1740#define BLKDEV_MAJOR_MAX 512
1741#else
1742#define BLKDEV_MAJOR_MAX 0
1743#endif
1744
1745struct blk_holder_ops {
1746 void (*mark_dead)(struct block_device *bdev, bool surprise);
1747
1748 /*
1749 * Sync the file system mounted on the block device.
1750 */
1751 void (*sync)(struct block_device *bdev);
1752
1753 /*
1754 * Freeze the file system mounted on the block device.
1755 */
1756 int (*freeze)(struct block_device *bdev);
1757
1758 /*
1759 * Thaw the file system mounted on the block device.
1760 */
1761 int (*thaw)(struct block_device *bdev);
1762};
1763
1764/*
1765 * For filesystems using @fs_holder_ops, the @holder argument passed to
1766 * helpers used to open and claim block devices via
1767 * bd_prepare_to_claim() must point to a superblock.
1768 */
1769extern const struct blk_holder_ops fs_holder_ops;
1770
1771/*
1772 * Return the correct open flags for blkdev_get_by_* for super block flags
1773 * as stored in sb->s_flags.
1774 */
1775#define sb_open_mode(flags) \
1776 (BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \
1777 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE))
1778
1779struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
1780 const struct blk_holder_ops *hops);
1781struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode,
1782 void *holder, const struct blk_holder_ops *hops);
1783int bd_prepare_to_claim(struct block_device *bdev, void *holder,
1784 const struct blk_holder_ops *hops);
1785void bd_abort_claiming(struct block_device *bdev, void *holder);
1786
1787struct block_device *I_BDEV(struct inode *inode);
1788struct block_device *file_bdev(struct file *bdev_file);
1789bool disk_live(struct gendisk *disk);
1790unsigned int block_size(struct block_device *bdev);
1791
1792#ifdef CONFIG_BLOCK
1793void invalidate_bdev(struct block_device *bdev);
1794int sync_blockdev(struct block_device *bdev);
1795int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
1796int sync_blockdev_nowait(struct block_device *bdev);
1797void sync_bdevs(bool wait);
1798void bdev_statx(const struct path *path, struct kstat *stat, u32 request_mask);
1799void printk_all_partitions(void);
1800int __init early_lookup_bdev(const char *pathname, dev_t *dev);
1801#else
1802static inline void invalidate_bdev(struct block_device *bdev)
1803{
1804}
1805static inline int sync_blockdev(struct block_device *bdev)
1806{
1807 return 0;
1808}
1809static inline int sync_blockdev_nowait(struct block_device *bdev)
1810{
1811 return 0;
1812}
1813static inline void sync_bdevs(bool wait)
1814{
1815}
1816static inline void bdev_statx(const struct path *path, struct kstat *stat,
1817 u32 request_mask)
1818{
1819}
1820static inline void printk_all_partitions(void)
1821{
1822}
1823static inline int early_lookup_bdev(const char *pathname, dev_t *dev)
1824{
1825 return -EINVAL;
1826}
1827#endif /* CONFIG_BLOCK */
1828
1829int bdev_freeze(struct block_device *bdev);
1830int bdev_thaw(struct block_device *bdev);
1831void bdev_fput(struct file *bdev_file);
1832
1833struct io_comp_batch {
1834 struct rq_list req_list;
1835 bool need_ts;
1836 void (*complete)(struct io_comp_batch *);
1837 void *poll_ctx;
1838};
1839
1840static inline bool blk_atomic_write_start_sect_aligned(sector_t sector,
1841 struct queue_limits *limits)
1842{
1843 unsigned int alignment = max(limits->atomic_write_hw_unit_min,
1844 limits->atomic_write_hw_boundary);
1845
1846 return IS_ALIGNED(sector, alignment >> SECTOR_SHIFT);
1847}
1848
1849static inline bool bdev_can_atomic_write(struct block_device *bdev)
1850{
1851 struct request_queue *bd_queue = bdev->bd_queue;
1852 struct queue_limits *limits = &bd_queue->limits;
1853
1854 if (!limits->atomic_write_unit_min)
1855 return false;
1856
1857 if (bdev_is_partition(bdev))
1858 return blk_atomic_write_start_sect_aligned(bdev->bd_start_sect,
1859 limits);
1860
1861 return true;
1862}
1863
1864static inline unsigned int
1865bdev_atomic_write_unit_min_bytes(struct block_device *bdev)
1866{
1867 if (!bdev_can_atomic_write(bdev))
1868 return 0;
1869 return queue_atomic_write_unit_min_bytes(bdev_get_queue(bdev));
1870}
1871
1872static inline unsigned int
1873bdev_atomic_write_unit_max_bytes(struct block_device *bdev)
1874{
1875 if (!bdev_can_atomic_write(bdev))
1876 return 0;
1877 return queue_atomic_write_unit_max_bytes(bdev_get_queue(bdev));
1878}
1879
1880static inline int bio_split_rw_at(struct bio *bio,
1881 const struct queue_limits *lim,
1882 unsigned *segs, unsigned max_bytes)
1883{
1884 return bio_split_io_at(bio, lim, segs, max_bytes, lim->dma_alignment);
1885}
1886
1887/*
1888 * Maximum contiguous integrity buffer allocation.
1889 */
1890#define BLK_INTEGRITY_MAX_SIZE SZ_2M
1891
1892/*
1893 * Maximum size of I/O that needs a block layer integrity buffer. Limited
1894 * by the number of intervals for which we can fit the integrity buffer into
1895 * the buffer size. Because the buffer is a single segment it is also limited
1896 * by the maximum segment size.
1897 */
1898static inline unsigned int max_integrity_io_size(struct queue_limits *lim)
1899{
1900 return min_t(unsigned int, lim->max_segment_size,
1901 (BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size) <<
1902 lim->integrity.interval_exp);
1903}
1904
1905#define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { }
1906
1907#endif /* _LINUX_BLKDEV_H */