1 #ifndef BLK_INTERNAL_H 2 #define BLK_INTERNAL_H 3 4 #include <linux/idr.h> 5 #include <linux/blk-mq.h> 6 #include "blk-mq.h" 7 8 /* Amount of time in which a process may batch requests */ 9 #define BLK_BATCH_TIME (HZ/50UL) 10 11 /* Number of requests a "batching" process may submit */ 12 #define BLK_BATCH_REQ 32 13 14 /* Max future timer expiry for timeouts */ 15 #define BLK_MAX_TIMEOUT (5 * HZ) 16 17 struct blk_flush_queue { 18 unsigned int flush_queue_delayed:1; 19 unsigned int flush_pending_idx:1; 20 unsigned int flush_running_idx:1; 21 unsigned long flush_pending_since; 22 struct list_head flush_queue[2]; 23 struct list_head flush_data_in_flight; 24 struct request *flush_rq; 25 spinlock_t mq_flush_lock; 26 }; 27 28 extern struct kmem_cache *blk_requestq_cachep; 29 extern struct kmem_cache *request_cachep; 30 extern struct kobj_type blk_queue_ktype; 31 extern struct ida blk_queue_ida; 32 33 static inline struct blk_flush_queue *blk_get_flush_queue( 34 struct request_queue *q, struct blk_mq_ctx *ctx) 35 { 36 struct blk_mq_hw_ctx *hctx; 37 38 if (!q->mq_ops) 39 return q->fq; 40 41 hctx = q->mq_ops->map_queue(q, ctx->cpu); 42 43 return hctx->fq; 44 } 45 46 static inline void __blk_get_queue(struct request_queue *q) 47 { 48 kobject_get(&q->kobj); 49 } 50 51 struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, 52 int node, int cmd_size); 53 void blk_free_flush_queue(struct blk_flush_queue *q); 54 55 int blk_init_rl(struct request_list *rl, struct request_queue *q, 56 gfp_t gfp_mask); 57 void blk_exit_rl(struct request_list *rl); 58 void init_request_from_bio(struct request *req, struct bio *bio); 59 void blk_rq_bio_prep(struct request_queue *q, struct request *rq, 60 struct bio *bio); 61 int blk_rq_append_bio(struct request_queue *q, struct request *rq, 62 struct bio *bio); 63 void blk_queue_bypass_start(struct request_queue *q); 64 void blk_queue_bypass_end(struct request_queue *q); 65 void blk_dequeue_request(struct request *rq); 66 void __blk_queue_free_tags(struct request_queue *q); 67 bool __blk_end_bidi_request(struct request *rq, int error, 68 unsigned int nr_bytes, unsigned int bidi_bytes); 69 70 void blk_rq_timed_out_timer(unsigned long data); 71 unsigned long blk_rq_timeout(unsigned long timeout); 72 void blk_add_timer(struct request *req); 73 void blk_delete_timer(struct request *); 74 75 76 bool bio_attempt_front_merge(struct request_queue *q, struct request *req, 77 struct bio *bio); 78 bool bio_attempt_back_merge(struct request_queue *q, struct request *req, 79 struct bio *bio); 80 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 81 unsigned int *request_count); 82 83 void blk_account_io_start(struct request *req, bool new_io); 84 void blk_account_io_completion(struct request *req, unsigned int bytes); 85 void blk_account_io_done(struct request *req); 86 87 /* 88 * Internal atomic flags for request handling 89 */ 90 enum rq_atomic_flags { 91 REQ_ATOM_COMPLETE = 0, 92 REQ_ATOM_STARTED, 93 }; 94 95 /* 96 * EH timer and IO completion will both attempt to 'grab' the request, make 97 * sure that only one of them succeeds 98 */ 99 static inline int blk_mark_rq_complete(struct request *rq) 100 { 101 return test_and_set_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); 102 } 103 104 static inline void blk_clear_rq_complete(struct request *rq) 105 { 106 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); 107 } 108 109 /* 110 * Internal elevator interface 111 */ 112 #define ELV_ON_HASH(rq) ((rq)->cmd_flags & REQ_HASHED) 113 114 void blk_insert_flush(struct request *rq); 115 116 static inline struct request *__elv_next_request(struct request_queue *q) 117 { 118 struct request *rq; 119 struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); 120 121 while (1) { 122 if (!list_empty(&q->queue_head)) { 123 rq = list_entry_rq(q->queue_head.next); 124 return rq; 125 } 126 127 /* 128 * Flush request is running and flush request isn't queueable 129 * in the drive, we can hold the queue till flush request is 130 * finished. Even we don't do this, driver can't dispatch next 131 * requests and will requeue them. And this can improve 132 * throughput too. For example, we have request flush1, write1, 133 * flush 2. flush1 is dispatched, then queue is hold, write1 134 * isn't inserted to queue. After flush1 is finished, flush2 135 * will be dispatched. Since disk cache is already clean, 136 * flush2 will be finished very soon, so looks like flush2 is 137 * folded to flush1. 138 * Since the queue is hold, a flag is set to indicate the queue 139 * should be restarted later. Please see flush_end_io() for 140 * details. 141 */ 142 if (fq->flush_pending_idx != fq->flush_running_idx && 143 !queue_flush_queueable(q)) { 144 fq->flush_queue_delayed = 1; 145 return NULL; 146 } 147 if (unlikely(blk_queue_bypass(q)) || 148 !q->elevator->type->ops.elevator_dispatch_fn(q, 0)) 149 return NULL; 150 } 151 } 152 153 static inline void elv_activate_rq(struct request_queue *q, struct request *rq) 154 { 155 struct elevator_queue *e = q->elevator; 156 157 if (e->type->ops.elevator_activate_req_fn) 158 e->type->ops.elevator_activate_req_fn(q, rq); 159 } 160 161 static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq) 162 { 163 struct elevator_queue *e = q->elevator; 164 165 if (e->type->ops.elevator_deactivate_req_fn) 166 e->type->ops.elevator_deactivate_req_fn(q, rq); 167 } 168 169 #ifdef CONFIG_FAIL_IO_TIMEOUT 170 int blk_should_fake_timeout(struct request_queue *); 171 ssize_t part_timeout_show(struct device *, struct device_attribute *, char *); 172 ssize_t part_timeout_store(struct device *, struct device_attribute *, 173 const char *, size_t); 174 #else 175 static inline int blk_should_fake_timeout(struct request_queue *q) 176 { 177 return 0; 178 } 179 #endif 180 181 int ll_back_merge_fn(struct request_queue *q, struct request *req, 182 struct bio *bio); 183 int ll_front_merge_fn(struct request_queue *q, struct request *req, 184 struct bio *bio); 185 int attempt_back_merge(struct request_queue *q, struct request *rq); 186 int attempt_front_merge(struct request_queue *q, struct request *rq); 187 int blk_attempt_req_merge(struct request_queue *q, struct request *rq, 188 struct request *next); 189 void blk_recalc_rq_segments(struct request *rq); 190 void blk_rq_set_mixed_merge(struct request *rq); 191 bool blk_rq_merge_ok(struct request *rq, struct bio *bio); 192 int blk_try_merge(struct request *rq, struct bio *bio); 193 194 void blk_queue_congestion_threshold(struct request_queue *q); 195 196 void __blk_run_queue_uncond(struct request_queue *q); 197 198 int blk_dev_init(void); 199 200 201 /* 202 * Return the threshold (number of used requests) at which the queue is 203 * considered to be congested. It include a little hysteresis to keep the 204 * context switch rate down. 205 */ 206 static inline int queue_congestion_on_threshold(struct request_queue *q) 207 { 208 return q->nr_congestion_on; 209 } 210 211 /* 212 * The threshold at which a queue is considered to be uncongested 213 */ 214 static inline int queue_congestion_off_threshold(struct request_queue *q) 215 { 216 return q->nr_congestion_off; 217 } 218 219 extern int blk_update_nr_requests(struct request_queue *, unsigned int); 220 221 /* 222 * Contribute to IO statistics IFF: 223 * 224 * a) it's attached to a gendisk, and 225 * b) the queue had IO stats enabled when this request was started, and 226 * c) it's a file system request 227 */ 228 static inline int blk_do_io_stat(struct request *rq) 229 { 230 return rq->rq_disk && 231 (rq->cmd_flags & REQ_IO_STAT) && 232 (rq->cmd_type == REQ_TYPE_FS); 233 } 234 235 /* 236 * Internal io_context interface 237 */ 238 void get_io_context(struct io_context *ioc); 239 struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q); 240 struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q, 241 gfp_t gfp_mask); 242 void ioc_clear_queue(struct request_queue *q); 243 244 int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node); 245 246 /** 247 * create_io_context - try to create task->io_context 248 * @gfp_mask: allocation mask 249 * @node: allocation node 250 * 251 * If %current->io_context is %NULL, allocate a new io_context and install 252 * it. Returns the current %current->io_context which may be %NULL if 253 * allocation failed. 254 * 255 * Note that this function can't be called with IRQ disabled because 256 * task_lock which protects %current->io_context is IRQ-unsafe. 257 */ 258 static inline struct io_context *create_io_context(gfp_t gfp_mask, int node) 259 { 260 WARN_ON_ONCE(irqs_disabled()); 261 if (unlikely(!current->io_context)) 262 create_task_io_context(current, gfp_mask, node); 263 return current->io_context; 264 } 265 266 /* 267 * Internal throttling interface 268 */ 269 #ifdef CONFIG_BLK_DEV_THROTTLING 270 extern bool blk_throtl_bio(struct request_queue *q, struct bio *bio); 271 extern void blk_throtl_drain(struct request_queue *q); 272 extern int blk_throtl_init(struct request_queue *q); 273 extern void blk_throtl_exit(struct request_queue *q); 274 #else /* CONFIG_BLK_DEV_THROTTLING */ 275 static inline bool blk_throtl_bio(struct request_queue *q, struct bio *bio) 276 { 277 return false; 278 } 279 static inline void blk_throtl_drain(struct request_queue *q) { } 280 static inline int blk_throtl_init(struct request_queue *q) { return 0; } 281 static inline void blk_throtl_exit(struct request_queue *q) { } 282 #endif /* CONFIG_BLK_DEV_THROTTLING */ 283 284 #endif /* BLK_INTERNAL_H */ 285