1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef BLK_INTERNAL_H 3 #define BLK_INTERNAL_H 4 5 #include <linux/idr.h> 6 #include <linux/blk-mq.h> 7 #include "blk-mq.h" 8 9 /* Amount of time in which a process may batch requests */ 10 #define BLK_BATCH_TIME (HZ/50UL) 11 12 /* Number of requests a "batching" process may submit */ 13 #define BLK_BATCH_REQ 32 14 15 /* Max future timer expiry for timeouts */ 16 #define BLK_MAX_TIMEOUT (5 * HZ) 17 18 #ifdef CONFIG_DEBUG_FS 19 extern struct dentry *blk_debugfs_root; 20 #endif 21 22 struct blk_flush_queue { 23 unsigned int flush_queue_delayed:1; 24 unsigned int flush_pending_idx:1; 25 unsigned int flush_running_idx:1; 26 unsigned long flush_pending_since; 27 struct list_head flush_queue[2]; 28 struct list_head flush_data_in_flight; 29 struct request *flush_rq; 30 31 /* 32 * flush_rq shares tag with this rq, both can't be active 33 * at the same time 34 */ 35 struct request *orig_rq; 36 spinlock_t mq_flush_lock; 37 }; 38 39 extern struct kmem_cache *blk_requestq_cachep; 40 extern struct kmem_cache *request_cachep; 41 extern struct kobj_type blk_queue_ktype; 42 extern struct ida blk_queue_ida; 43 44 static inline struct blk_flush_queue *blk_get_flush_queue( 45 struct request_queue *q, struct blk_mq_ctx *ctx) 46 { 47 if (q->mq_ops) 48 return blk_mq_map_queue(q, ctx->cpu)->fq; 49 return q->fq; 50 } 51 52 static inline void __blk_get_queue(struct request_queue *q) 53 { 54 kobject_get(&q->kobj); 55 } 56 57 struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, 58 int node, int cmd_size); 59 void blk_free_flush_queue(struct blk_flush_queue *q); 60 61 int blk_init_rl(struct request_list *rl, struct request_queue *q, 62 gfp_t gfp_mask); 63 void blk_exit_rl(struct request_queue *q, struct request_list *rl); 64 void blk_rq_bio_prep(struct request_queue *q, struct request *rq, 65 struct bio *bio); 66 void blk_queue_bypass_start(struct request_queue *q); 67 void blk_queue_bypass_end(struct request_queue *q); 68 void __blk_queue_free_tags(struct request_queue *q); 69 void blk_freeze_queue(struct request_queue *q); 70 71 static inline void blk_queue_enter_live(struct request_queue *q) 72 { 73 /* 74 * Given that running in generic_make_request() context 75 * guarantees that a live reference against q_usage_counter has 76 * been established, further references under that same context 77 * need not check that the queue has been frozen (marked dead). 78 */ 79 percpu_ref_get(&q->q_usage_counter); 80 } 81 82 #ifdef CONFIG_BLK_DEV_INTEGRITY 83 void blk_flush_integrity(void); 84 bool __bio_integrity_endio(struct bio *); 85 static inline bool bio_integrity_endio(struct bio *bio) 86 { 87 if (bio_integrity(bio)) 88 return __bio_integrity_endio(bio); 89 return true; 90 } 91 #else 92 static inline void blk_flush_integrity(void) 93 { 94 } 95 static inline bool bio_integrity_endio(struct bio *bio) 96 { 97 return true; 98 } 99 #endif 100 101 void blk_timeout_work(struct work_struct *work); 102 unsigned long blk_rq_timeout(unsigned long timeout); 103 void blk_add_timer(struct request *req); 104 void blk_delete_timer(struct request *); 105 106 107 bool bio_attempt_front_merge(struct request_queue *q, struct request *req, 108 struct bio *bio); 109 bool bio_attempt_back_merge(struct request_queue *q, struct request *req, 110 struct bio *bio); 111 bool bio_attempt_discard_merge(struct request_queue *q, struct request *req, 112 struct bio *bio); 113 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 114 unsigned int *request_count, 115 struct request **same_queue_rq); 116 unsigned int blk_plug_queued_count(struct request_queue *q); 117 118 void blk_account_io_start(struct request *req, bool new_io); 119 void blk_account_io_completion(struct request *req, unsigned int bytes); 120 void blk_account_io_done(struct request *req); 121 122 /* 123 * EH timer and IO completion will both attempt to 'grab' the request, make 124 * sure that only one of them succeeds. Steal the bottom bit of the 125 * __deadline field for this. 126 */ 127 static inline int blk_mark_rq_complete(struct request *rq) 128 { 129 return test_and_set_bit(0, &rq->__deadline); 130 } 131 132 static inline void blk_clear_rq_complete(struct request *rq) 133 { 134 clear_bit(0, &rq->__deadline); 135 } 136 137 static inline bool blk_rq_is_complete(struct request *rq) 138 { 139 return test_bit(0, &rq->__deadline); 140 } 141 142 /* 143 * Internal elevator interface 144 */ 145 #define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED) 146 147 void blk_insert_flush(struct request *rq); 148 149 static inline void elv_activate_rq(struct request_queue *q, struct request *rq) 150 { 151 struct elevator_queue *e = q->elevator; 152 153 if (e->type->ops.sq.elevator_activate_req_fn) 154 e->type->ops.sq.elevator_activate_req_fn(q, rq); 155 } 156 157 static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq) 158 { 159 struct elevator_queue *e = q->elevator; 160 161 if (e->type->ops.sq.elevator_deactivate_req_fn) 162 e->type->ops.sq.elevator_deactivate_req_fn(q, rq); 163 } 164 165 int elv_register_queue(struct request_queue *q); 166 void elv_unregister_queue(struct request_queue *q); 167 168 struct hd_struct *__disk_get_part(struct gendisk *disk, int partno); 169 170 #ifdef CONFIG_FAIL_IO_TIMEOUT 171 int blk_should_fake_timeout(struct request_queue *); 172 ssize_t part_timeout_show(struct device *, struct device_attribute *, char *); 173 ssize_t part_timeout_store(struct device *, struct device_attribute *, 174 const char *, size_t); 175 #else 176 static inline int blk_should_fake_timeout(struct request_queue *q) 177 { 178 return 0; 179 } 180 #endif 181 182 int ll_back_merge_fn(struct request_queue *q, struct request *req, 183 struct bio *bio); 184 int ll_front_merge_fn(struct request_queue *q, struct request *req, 185 struct bio *bio); 186 struct request *attempt_back_merge(struct request_queue *q, struct request *rq); 187 struct request *attempt_front_merge(struct request_queue *q, struct request *rq); 188 int blk_attempt_req_merge(struct request_queue *q, struct request *rq, 189 struct request *next); 190 void blk_recalc_rq_segments(struct request *rq); 191 void blk_rq_set_mixed_merge(struct request *rq); 192 bool blk_rq_merge_ok(struct request *rq, struct bio *bio); 193 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio); 194 195 void blk_queue_congestion_threshold(struct request_queue *q); 196 197 int blk_dev_init(void); 198 199 200 /* 201 * Return the threshold (number of used requests) at which the queue is 202 * considered to be congested. It include a little hysteresis to keep the 203 * context switch rate down. 204 */ 205 static inline int queue_congestion_on_threshold(struct request_queue *q) 206 { 207 return q->nr_congestion_on; 208 } 209 210 /* 211 * The threshold at which a queue is considered to be uncongested 212 */ 213 static inline int queue_congestion_off_threshold(struct request_queue *q) 214 { 215 return q->nr_congestion_off; 216 } 217 218 extern int blk_update_nr_requests(struct request_queue *, unsigned int); 219 220 /* 221 * Contribute to IO statistics IFF: 222 * 223 * a) it's attached to a gendisk, and 224 * b) the queue had IO stats enabled when this request was started, and 225 * c) it's a file system request 226 */ 227 static inline int blk_do_io_stat(struct request *rq) 228 { 229 return rq->rq_disk && 230 (rq->rq_flags & RQF_IO_STAT) && 231 !blk_rq_is_passthrough(rq); 232 } 233 234 static inline void req_set_nomerge(struct request_queue *q, struct request *req) 235 { 236 req->cmd_flags |= REQ_NOMERGE; 237 if (req == q->last_merge) 238 q->last_merge = NULL; 239 } 240 241 /* 242 * Steal a bit from this field for legacy IO path atomic IO marking. Note that 243 * setting the deadline clears the bottom bit, potentially clearing the 244 * completed bit. The user has to be OK with this (current ones are fine). 245 */ 246 static inline void blk_rq_set_deadline(struct request *rq, unsigned long time) 247 { 248 rq->__deadline = time & ~0x1UL; 249 } 250 251 static inline unsigned long blk_rq_deadline(struct request *rq) 252 { 253 return rq->__deadline & ~0x1UL; 254 } 255 256 /* 257 * Internal io_context interface 258 */ 259 void get_io_context(struct io_context *ioc); 260 struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q); 261 struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q, 262 gfp_t gfp_mask); 263 void ioc_clear_queue(struct request_queue *q); 264 265 int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node); 266 267 /** 268 * rq_ioc - determine io_context for request allocation 269 * @bio: request being allocated is for this bio (can be %NULL) 270 * 271 * Determine io_context to use for request allocation for @bio. May return 272 * %NULL if %current->io_context doesn't exist. 273 */ 274 static inline struct io_context *rq_ioc(struct bio *bio) 275 { 276 #ifdef CONFIG_BLK_CGROUP 277 if (bio && bio->bi_ioc) 278 return bio->bi_ioc; 279 #endif 280 return current->io_context; 281 } 282 283 /** 284 * create_io_context - try to create task->io_context 285 * @gfp_mask: allocation mask 286 * @node: allocation node 287 * 288 * If %current->io_context is %NULL, allocate a new io_context and install 289 * it. Returns the current %current->io_context which may be %NULL if 290 * allocation failed. 291 * 292 * Note that this function can't be called with IRQ disabled because 293 * task_lock which protects %current->io_context is IRQ-unsafe. 294 */ 295 static inline struct io_context *create_io_context(gfp_t gfp_mask, int node) 296 { 297 WARN_ON_ONCE(irqs_disabled()); 298 if (unlikely(!current->io_context)) 299 create_task_io_context(current, gfp_mask, node); 300 return current->io_context; 301 } 302 303 /* 304 * Internal throttling interface 305 */ 306 #ifdef CONFIG_BLK_DEV_THROTTLING 307 extern void blk_throtl_drain(struct request_queue *q); 308 extern int blk_throtl_init(struct request_queue *q); 309 extern void blk_throtl_exit(struct request_queue *q); 310 extern void blk_throtl_register_queue(struct request_queue *q); 311 #else /* CONFIG_BLK_DEV_THROTTLING */ 312 static inline void blk_throtl_drain(struct request_queue *q) { } 313 static inline int blk_throtl_init(struct request_queue *q) { return 0; } 314 static inline void blk_throtl_exit(struct request_queue *q) { } 315 static inline void blk_throtl_register_queue(struct request_queue *q) { } 316 #endif /* CONFIG_BLK_DEV_THROTTLING */ 317 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW 318 extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page); 319 extern ssize_t blk_throtl_sample_time_store(struct request_queue *q, 320 const char *page, size_t count); 321 extern void blk_throtl_bio_endio(struct bio *bio); 322 extern void blk_throtl_stat_add(struct request *rq, u64 time); 323 #else 324 static inline void blk_throtl_bio_endio(struct bio *bio) { } 325 static inline void blk_throtl_stat_add(struct request *rq, u64 time) { } 326 #endif 327 328 #ifdef CONFIG_BOUNCE 329 extern int init_emergency_isa_pool(void); 330 extern void blk_queue_bounce(struct request_queue *q, struct bio **bio); 331 #else 332 static inline int init_emergency_isa_pool(void) 333 { 334 return 0; 335 } 336 static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio) 337 { 338 } 339 #endif /* CONFIG_BOUNCE */ 340 341 extern void blk_drain_queue(struct request_queue *q); 342 343 #endif /* BLK_INTERNAL_H */ 344