1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef BLK_INTERNAL_H 3 #define BLK_INTERNAL_H 4 5 #include <linux/idr.h> 6 #include <linux/blk-mq.h> 7 #include <linux/part_stat.h> 8 #include <linux/blk-crypto.h> 9 #include <linux/memblock.h> /* for max_pfn/max_low_pfn */ 10 #include <xen/xen.h> 11 #include "blk-crypto-internal.h" 12 #include "blk-mq.h" 13 #include "blk-mq-sched.h" 14 15 struct elevator_type; 16 17 /* Max future timer expiry for timeouts */ 18 #define BLK_MAX_TIMEOUT (5 * HZ) 19 20 extern struct dentry *blk_debugfs_root; 21 22 struct blk_flush_queue { 23 unsigned int flush_pending_idx:1; 24 unsigned int flush_running_idx:1; 25 blk_status_t rq_status; 26 unsigned long flush_pending_since; 27 struct list_head flush_queue[2]; 28 struct list_head flush_data_in_flight; 29 struct request *flush_rq; 30 31 spinlock_t mq_flush_lock; 32 }; 33 34 extern struct kmem_cache *blk_requestq_cachep; 35 extern struct kobj_type blk_queue_ktype; 36 extern struct ida blk_queue_ida; 37 38 static inline struct blk_flush_queue * 39 blk_get_flush_queue(struct request_queue *q, struct blk_mq_ctx *ctx) 40 { 41 return blk_mq_map_queue(q, REQ_OP_FLUSH, ctx)->fq; 42 } 43 44 static inline void __blk_get_queue(struct request_queue *q) 45 { 46 kobject_get(&q->kobj); 47 } 48 49 bool is_flush_rq(struct request *req); 50 51 struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size, 52 gfp_t flags); 53 void blk_free_flush_queue(struct blk_flush_queue *q); 54 55 void blk_freeze_queue(struct request_queue *q); 56 void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic); 57 void blk_queue_start_drain(struct request_queue *q); 58 int __bio_queue_enter(struct request_queue *q, struct bio *bio); 59 bool submit_bio_checks(struct bio *bio); 60 61 static inline bool blk_try_enter_queue(struct request_queue *q, bool pm) 62 { 63 rcu_read_lock(); 64 if (!percpu_ref_tryget_live_rcu(&q->q_usage_counter)) 65 goto fail; 66 67 /* 68 * The code that increments the pm_only counter must ensure that the 69 * counter is globally visible before the queue is unfrozen. 70 */ 71 if (blk_queue_pm_only(q) && 72 (!pm || queue_rpm_status(q) == RPM_SUSPENDED)) 73 goto fail_put; 74 75 rcu_read_unlock(); 76 return true; 77 78 fail_put: 79 blk_queue_exit(q); 80 fail: 81 rcu_read_unlock(); 82 return false; 83 } 84 85 static inline int bio_queue_enter(struct bio *bio) 86 { 87 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 88 89 if (blk_try_enter_queue(q, false)) 90 return 0; 91 return __bio_queue_enter(q, bio); 92 } 93 94 #define BIO_INLINE_VECS 4 95 struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, 96 gfp_t gfp_mask); 97 void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs); 98 99 static inline bool biovec_phys_mergeable(struct request_queue *q, 100 struct bio_vec *vec1, struct bio_vec *vec2) 101 { 102 unsigned long mask = queue_segment_boundary(q); 103 phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset; 104 phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset; 105 106 if (addr1 + vec1->bv_len != addr2) 107 return false; 108 if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) 109 return false; 110 if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) 111 return false; 112 return true; 113 } 114 115 static inline bool __bvec_gap_to_prev(struct request_queue *q, 116 struct bio_vec *bprv, unsigned int offset) 117 { 118 return (offset & queue_virt_boundary(q)) || 119 ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); 120 } 121 122 /* 123 * Check if adding a bio_vec after bprv with offset would create a gap in 124 * the SG list. Most drivers don't care about this, but some do. 125 */ 126 static inline bool bvec_gap_to_prev(struct request_queue *q, 127 struct bio_vec *bprv, unsigned int offset) 128 { 129 if (!queue_virt_boundary(q)) 130 return false; 131 return __bvec_gap_to_prev(q, bprv, offset); 132 } 133 134 static inline bool rq_mergeable(struct request *rq) 135 { 136 if (blk_rq_is_passthrough(rq)) 137 return false; 138 139 if (req_op(rq) == REQ_OP_FLUSH) 140 return false; 141 142 if (req_op(rq) == REQ_OP_WRITE_ZEROES) 143 return false; 144 145 if (req_op(rq) == REQ_OP_ZONE_APPEND) 146 return false; 147 148 if (rq->cmd_flags & REQ_NOMERGE_FLAGS) 149 return false; 150 if (rq->rq_flags & RQF_NOMERGE_FLAGS) 151 return false; 152 153 return true; 154 } 155 156 /* 157 * There are two different ways to handle DISCARD merges: 158 * 1) If max_discard_segments > 1, the driver treats every bio as a range and 159 * send the bios to controller together. The ranges don't need to be 160 * contiguous. 161 * 2) Otherwise, the request will be normal read/write requests. The ranges 162 * need to be contiguous. 163 */ 164 static inline bool blk_discard_mergable(struct request *req) 165 { 166 if (req_op(req) == REQ_OP_DISCARD && 167 queue_max_discard_segments(req->q) > 1) 168 return true; 169 return false; 170 } 171 172 #ifdef CONFIG_BLK_DEV_INTEGRITY 173 void blk_flush_integrity(void); 174 bool __bio_integrity_endio(struct bio *); 175 void bio_integrity_free(struct bio *bio); 176 static inline bool bio_integrity_endio(struct bio *bio) 177 { 178 if (bio_integrity(bio)) 179 return __bio_integrity_endio(bio); 180 return true; 181 } 182 183 bool blk_integrity_merge_rq(struct request_queue *, struct request *, 184 struct request *); 185 bool blk_integrity_merge_bio(struct request_queue *, struct request *, 186 struct bio *); 187 188 static inline bool integrity_req_gap_back_merge(struct request *req, 189 struct bio *next) 190 { 191 struct bio_integrity_payload *bip = bio_integrity(req->bio); 192 struct bio_integrity_payload *bip_next = bio_integrity(next); 193 194 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], 195 bip_next->bip_vec[0].bv_offset); 196 } 197 198 static inline bool integrity_req_gap_front_merge(struct request *req, 199 struct bio *bio) 200 { 201 struct bio_integrity_payload *bip = bio_integrity(bio); 202 struct bio_integrity_payload *bip_next = bio_integrity(req->bio); 203 204 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], 205 bip_next->bip_vec[0].bv_offset); 206 } 207 208 int blk_integrity_add(struct gendisk *disk); 209 void blk_integrity_del(struct gendisk *); 210 #else /* CONFIG_BLK_DEV_INTEGRITY */ 211 static inline bool blk_integrity_merge_rq(struct request_queue *rq, 212 struct request *r1, struct request *r2) 213 { 214 return true; 215 } 216 static inline bool blk_integrity_merge_bio(struct request_queue *rq, 217 struct request *r, struct bio *b) 218 { 219 return true; 220 } 221 static inline bool integrity_req_gap_back_merge(struct request *req, 222 struct bio *next) 223 { 224 return false; 225 } 226 static inline bool integrity_req_gap_front_merge(struct request *req, 227 struct bio *bio) 228 { 229 return false; 230 } 231 232 static inline void blk_flush_integrity(void) 233 { 234 } 235 static inline bool bio_integrity_endio(struct bio *bio) 236 { 237 return true; 238 } 239 static inline void bio_integrity_free(struct bio *bio) 240 { 241 } 242 static inline int blk_integrity_add(struct gendisk *disk) 243 { 244 return 0; 245 } 246 static inline void blk_integrity_del(struct gendisk *disk) 247 { 248 } 249 #endif /* CONFIG_BLK_DEV_INTEGRITY */ 250 251 unsigned long blk_rq_timeout(unsigned long timeout); 252 void blk_add_timer(struct request *req); 253 void blk_print_req_error(struct request *req, blk_status_t status); 254 255 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 256 unsigned int nr_segs, bool *same_queue_rq); 257 bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, 258 struct bio *bio, unsigned int nr_segs); 259 260 void __blk_account_io_start(struct request *req); 261 void __blk_account_io_done(struct request *req, u64 now); 262 263 /* 264 * Plug flush limits 265 */ 266 #define BLK_MAX_REQUEST_COUNT 32 267 #define BLK_PLUG_FLUSH_SIZE (128 * 1024) 268 269 /* 270 * Internal elevator interface 271 */ 272 #define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED) 273 274 bool blk_insert_flush(struct request *rq); 275 276 int elevator_switch_mq(struct request_queue *q, 277 struct elevator_type *new_e); 278 void __elevator_exit(struct request_queue *, struct elevator_queue *); 279 int elv_register_queue(struct request_queue *q, bool uevent); 280 void elv_unregister_queue(struct request_queue *q); 281 282 static inline void elevator_exit(struct request_queue *q, 283 struct elevator_queue *e) 284 { 285 lockdep_assert_held(&q->sysfs_lock); 286 287 blk_mq_sched_free_rqs(q); 288 __elevator_exit(q, e); 289 } 290 291 ssize_t part_size_show(struct device *dev, struct device_attribute *attr, 292 char *buf); 293 ssize_t part_stat_show(struct device *dev, struct device_attribute *attr, 294 char *buf); 295 ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr, 296 char *buf); 297 ssize_t part_fail_show(struct device *dev, struct device_attribute *attr, 298 char *buf); 299 ssize_t part_fail_store(struct device *dev, struct device_attribute *attr, 300 const char *buf, size_t count); 301 ssize_t part_timeout_show(struct device *, struct device_attribute *, char *); 302 ssize_t part_timeout_store(struct device *, struct device_attribute *, 303 const char *, size_t); 304 305 static inline bool blk_may_split(struct request_queue *q, struct bio *bio) 306 { 307 switch (bio_op(bio)) { 308 case REQ_OP_DISCARD: 309 case REQ_OP_SECURE_ERASE: 310 case REQ_OP_WRITE_ZEROES: 311 case REQ_OP_WRITE_SAME: 312 return true; /* non-trivial splitting decisions */ 313 default: 314 break; 315 } 316 317 /* 318 * All drivers must accept single-segments bios that are <= PAGE_SIZE. 319 * This is a quick and dirty check that relies on the fact that 320 * bi_io_vec[0] is always valid if a bio has data. The check might 321 * lead to occasional false negatives when bios are cloned, but compared 322 * to the performance impact of cloned bios themselves the loop below 323 * doesn't matter anyway. 324 */ 325 return q->limits.chunk_sectors || bio->bi_vcnt != 1 || 326 bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE; 327 } 328 329 void __blk_queue_split(struct request_queue *q, struct bio **bio, 330 unsigned int *nr_segs); 331 int ll_back_merge_fn(struct request *req, struct bio *bio, 332 unsigned int nr_segs); 333 bool blk_attempt_req_merge(struct request_queue *q, struct request *rq, 334 struct request *next); 335 unsigned int blk_recalc_rq_segments(struct request *rq); 336 void blk_rq_set_mixed_merge(struct request *rq); 337 bool blk_rq_merge_ok(struct request *rq, struct bio *bio); 338 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio); 339 340 int blk_dev_init(void); 341 342 /* 343 * Contribute to IO statistics IFF: 344 * 345 * a) it's attached to a gendisk, and 346 * b) the queue had IO stats enabled when this request was started 347 */ 348 static inline bool blk_do_io_stat(struct request *rq) 349 { 350 return (rq->rq_flags & RQF_IO_STAT) && rq->rq_disk; 351 } 352 353 static inline void blk_account_io_done(struct request *req, u64 now) 354 { 355 /* 356 * Account IO completion. flush_rq isn't accounted as a 357 * normal IO on queueing nor completion. Accounting the 358 * containing request is enough. 359 */ 360 if (blk_do_io_stat(req) && req->part && 361 !(req->rq_flags & RQF_FLUSH_SEQ)) 362 __blk_account_io_done(req, now); 363 } 364 365 static inline void blk_account_io_start(struct request *req) 366 { 367 if (blk_do_io_stat(req)) 368 __blk_account_io_start(req); 369 } 370 371 static inline void req_set_nomerge(struct request_queue *q, struct request *req) 372 { 373 req->cmd_flags |= REQ_NOMERGE; 374 if (req == q->last_merge) 375 q->last_merge = NULL; 376 } 377 378 /* 379 * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size 380 * is defined as 'unsigned int', meantime it has to aligned to with logical 381 * block size which is the minimum accepted unit by hardware. 382 */ 383 static inline unsigned int bio_allowed_max_sectors(struct request_queue *q) 384 { 385 return round_down(UINT_MAX, queue_logical_block_size(q)) >> 9; 386 } 387 388 /* 389 * The max bio size which is aligned to q->limits.discard_granularity. This 390 * is a hint to split large discard bio in generic block layer, then if device 391 * driver needs to split the discard bio into smaller ones, their bi_size can 392 * be very probably and easily aligned to discard_granularity of the device's 393 * queue. 394 */ 395 static inline unsigned int bio_aligned_discard_max_sectors( 396 struct request_queue *q) 397 { 398 return round_down(UINT_MAX, q->limits.discard_granularity) >> 399 SECTOR_SHIFT; 400 } 401 402 /* 403 * Internal io_context interface 404 */ 405 void get_io_context(struct io_context *ioc); 406 struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q); 407 struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q, 408 gfp_t gfp_mask); 409 void ioc_clear_queue(struct request_queue *q); 410 411 int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node); 412 413 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW 414 extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page); 415 extern ssize_t blk_throtl_sample_time_store(struct request_queue *q, 416 const char *page, size_t count); 417 extern void blk_throtl_bio_endio(struct bio *bio); 418 extern void blk_throtl_stat_add(struct request *rq, u64 time); 419 #else 420 static inline void blk_throtl_bio_endio(struct bio *bio) { } 421 static inline void blk_throtl_stat_add(struct request *rq, u64 time) { } 422 #endif 423 424 void __blk_queue_bounce(struct request_queue *q, struct bio **bio); 425 426 static inline bool blk_queue_may_bounce(struct request_queue *q) 427 { 428 return IS_ENABLED(CONFIG_BOUNCE) && 429 q->limits.bounce == BLK_BOUNCE_HIGH && 430 max_low_pfn >= max_pfn; 431 } 432 433 static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio) 434 { 435 if (unlikely(blk_queue_may_bounce(q) && bio_has_data(*bio))) 436 __blk_queue_bounce(q, bio); 437 } 438 439 #ifdef CONFIG_BLK_CGROUP_IOLATENCY 440 extern int blk_iolatency_init(struct request_queue *q); 441 #else 442 static inline int blk_iolatency_init(struct request_queue *q) { return 0; } 443 #endif 444 445 struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp); 446 447 #ifdef CONFIG_BLK_DEV_ZONED 448 void blk_queue_free_zone_bitmaps(struct request_queue *q); 449 void blk_queue_clear_zone_settings(struct request_queue *q); 450 #else 451 static inline void blk_queue_free_zone_bitmaps(struct request_queue *q) {} 452 static inline void blk_queue_clear_zone_settings(struct request_queue *q) {} 453 #endif 454 455 int blk_alloc_ext_minor(void); 456 void blk_free_ext_minor(unsigned int minor); 457 #define ADDPART_FLAG_NONE 0 458 #define ADDPART_FLAG_RAID 1 459 #define ADDPART_FLAG_WHOLEDISK 2 460 int bdev_add_partition(struct gendisk *disk, int partno, sector_t start, 461 sector_t length); 462 int bdev_del_partition(struct gendisk *disk, int partno); 463 int bdev_resize_partition(struct gendisk *disk, int partno, sector_t start, 464 sector_t length); 465 466 int bio_add_hw_page(struct request_queue *q, struct bio *bio, 467 struct page *page, unsigned int len, unsigned int offset, 468 unsigned int max_sectors, bool *same_page); 469 470 struct request_queue *blk_alloc_queue(int node_id); 471 472 int disk_alloc_events(struct gendisk *disk); 473 void disk_add_events(struct gendisk *disk); 474 void disk_del_events(struct gendisk *disk); 475 void disk_release_events(struct gendisk *disk); 476 extern struct device_attribute dev_attr_events; 477 extern struct device_attribute dev_attr_events_async; 478 extern struct device_attribute dev_attr_events_poll_msecs; 479 480 static inline void bio_clear_polled(struct bio *bio) 481 { 482 /* can't support alloc cache if we turn off polling */ 483 bio_clear_flag(bio, BIO_PERCPU_CACHE); 484 bio->bi_opf &= ~REQ_POLLED; 485 } 486 487 long blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg); 488 long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg); 489 490 extern const struct address_space_operations def_blk_aops; 491 492 int disk_register_independent_access_ranges(struct gendisk *disk, 493 struct blk_independent_access_ranges *new_iars); 494 void disk_unregister_independent_access_ranges(struct gendisk *disk); 495 496 #endif /* BLK_INTERNAL_H */ 497