1 #ifndef BLK_INTERNAL_H 2 #define BLK_INTERNAL_H 3 4 #include <linux/idr.h> 5 #include <linux/blk-mq.h> 6 #include "blk-mq.h" 7 8 /* Amount of time in which a process may batch requests */ 9 #define BLK_BATCH_TIME (HZ/50UL) 10 11 /* Number of requests a "batching" process may submit */ 12 #define BLK_BATCH_REQ 32 13 14 /* Max future timer expiry for timeouts */ 15 #define BLK_MAX_TIMEOUT (5 * HZ) 16 17 struct blk_flush_queue { 18 unsigned int flush_queue_delayed:1; 19 unsigned int flush_pending_idx:1; 20 unsigned int flush_running_idx:1; 21 unsigned long flush_pending_since; 22 struct list_head flush_queue[2]; 23 struct list_head flush_data_in_flight; 24 struct request *flush_rq; 25 spinlock_t mq_flush_lock; 26 }; 27 28 extern struct kmem_cache *blk_requestq_cachep; 29 extern struct kmem_cache *request_cachep; 30 extern struct kobj_type blk_queue_ktype; 31 extern struct ida blk_queue_ida; 32 33 static inline struct blk_flush_queue *blk_get_flush_queue( 34 struct request_queue *q, struct blk_mq_ctx *ctx) 35 { 36 struct blk_mq_hw_ctx *hctx; 37 38 if (!q->mq_ops) 39 return q->fq; 40 41 hctx = q->mq_ops->map_queue(q, ctx->cpu); 42 43 return hctx->fq; 44 } 45 46 static inline void __blk_get_queue(struct request_queue *q) 47 { 48 kobject_get(&q->kobj); 49 } 50 51 struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, 52 int node, int cmd_size); 53 void blk_free_flush_queue(struct blk_flush_queue *q); 54 55 int blk_init_rl(struct request_list *rl, struct request_queue *q, 56 gfp_t gfp_mask); 57 void blk_exit_rl(struct request_list *rl); 58 void init_request_from_bio(struct request *req, struct bio *bio); 59 void blk_rq_bio_prep(struct request_queue *q, struct request *rq, 60 struct bio *bio); 61 int blk_rq_append_bio(struct request_queue *q, struct request *rq, 62 struct bio *bio); 63 void blk_queue_bypass_start(struct request_queue *q); 64 void blk_queue_bypass_end(struct request_queue *q); 65 void blk_dequeue_request(struct request *rq); 66 void __blk_queue_free_tags(struct request_queue *q); 67 bool __blk_end_bidi_request(struct request *rq, int error, 68 unsigned int nr_bytes, unsigned int bidi_bytes); 69 70 void blk_rq_timed_out_timer(unsigned long data); 71 unsigned long blk_rq_timeout(unsigned long timeout); 72 void blk_add_timer(struct request *req); 73 void blk_delete_timer(struct request *); 74 75 76 bool bio_attempt_front_merge(struct request_queue *q, struct request *req, 77 struct bio *bio); 78 bool bio_attempt_back_merge(struct request_queue *q, struct request *req, 79 struct bio *bio); 80 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 81 unsigned int *request_count, 82 struct request **same_queue_rq); 83 84 void blk_account_io_start(struct request *req, bool new_io); 85 void blk_account_io_completion(struct request *req, unsigned int bytes); 86 void blk_account_io_done(struct request *req); 87 88 /* 89 * Internal atomic flags for request handling 90 */ 91 enum rq_atomic_flags { 92 REQ_ATOM_COMPLETE = 0, 93 REQ_ATOM_STARTED, 94 }; 95 96 /* 97 * EH timer and IO completion will both attempt to 'grab' the request, make 98 * sure that only one of them succeeds 99 */ 100 static inline int blk_mark_rq_complete(struct request *rq) 101 { 102 return test_and_set_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); 103 } 104 105 static inline void blk_clear_rq_complete(struct request *rq) 106 { 107 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); 108 } 109 110 /* 111 * Internal elevator interface 112 */ 113 #define ELV_ON_HASH(rq) ((rq)->cmd_flags & REQ_HASHED) 114 115 void blk_insert_flush(struct request *rq); 116 117 static inline struct request *__elv_next_request(struct request_queue *q) 118 { 119 struct request *rq; 120 struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); 121 122 while (1) { 123 if (!list_empty(&q->queue_head)) { 124 rq = list_entry_rq(q->queue_head.next); 125 return rq; 126 } 127 128 /* 129 * Flush request is running and flush request isn't queueable 130 * in the drive, we can hold the queue till flush request is 131 * finished. Even we don't do this, driver can't dispatch next 132 * requests and will requeue them. And this can improve 133 * throughput too. For example, we have request flush1, write1, 134 * flush 2. flush1 is dispatched, then queue is hold, write1 135 * isn't inserted to queue. After flush1 is finished, flush2 136 * will be dispatched. Since disk cache is already clean, 137 * flush2 will be finished very soon, so looks like flush2 is 138 * folded to flush1. 139 * Since the queue is hold, a flag is set to indicate the queue 140 * should be restarted later. Please see flush_end_io() for 141 * details. 142 */ 143 if (fq->flush_pending_idx != fq->flush_running_idx && 144 !queue_flush_queueable(q)) { 145 fq->flush_queue_delayed = 1; 146 return NULL; 147 } 148 if (unlikely(blk_queue_bypass(q)) || 149 !q->elevator->type->ops.elevator_dispatch_fn(q, 0)) 150 return NULL; 151 } 152 } 153 154 static inline void elv_activate_rq(struct request_queue *q, struct request *rq) 155 { 156 struct elevator_queue *e = q->elevator; 157 158 if (e->type->ops.elevator_activate_req_fn) 159 e->type->ops.elevator_activate_req_fn(q, rq); 160 } 161 162 static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq) 163 { 164 struct elevator_queue *e = q->elevator; 165 166 if (e->type->ops.elevator_deactivate_req_fn) 167 e->type->ops.elevator_deactivate_req_fn(q, rq); 168 } 169 170 #ifdef CONFIG_FAIL_IO_TIMEOUT 171 int blk_should_fake_timeout(struct request_queue *); 172 ssize_t part_timeout_show(struct device *, struct device_attribute *, char *); 173 ssize_t part_timeout_store(struct device *, struct device_attribute *, 174 const char *, size_t); 175 #else 176 static inline int blk_should_fake_timeout(struct request_queue *q) 177 { 178 return 0; 179 } 180 #endif 181 182 int ll_back_merge_fn(struct request_queue *q, struct request *req, 183 struct bio *bio); 184 int ll_front_merge_fn(struct request_queue *q, struct request *req, 185 struct bio *bio); 186 int attempt_back_merge(struct request_queue *q, struct request *rq); 187 int attempt_front_merge(struct request_queue *q, struct request *rq); 188 int blk_attempt_req_merge(struct request_queue *q, struct request *rq, 189 struct request *next); 190 void blk_recalc_rq_segments(struct request *rq); 191 void blk_rq_set_mixed_merge(struct request *rq); 192 bool blk_rq_merge_ok(struct request *rq, struct bio *bio); 193 int blk_try_merge(struct request *rq, struct bio *bio); 194 195 void blk_queue_congestion_threshold(struct request_queue *q); 196 197 int blk_dev_init(void); 198 199 200 /* 201 * Return the threshold (number of used requests) at which the queue is 202 * considered to be congested. It include a little hysteresis to keep the 203 * context switch rate down. 204 */ 205 static inline int queue_congestion_on_threshold(struct request_queue *q) 206 { 207 return q->nr_congestion_on; 208 } 209 210 /* 211 * The threshold at which a queue is considered to be uncongested 212 */ 213 static inline int queue_congestion_off_threshold(struct request_queue *q) 214 { 215 return q->nr_congestion_off; 216 } 217 218 extern int blk_update_nr_requests(struct request_queue *, unsigned int); 219 220 /* 221 * Contribute to IO statistics IFF: 222 * 223 * a) it's attached to a gendisk, and 224 * b) the queue had IO stats enabled when this request was started, and 225 * c) it's a file system request 226 */ 227 static inline int blk_do_io_stat(struct request *rq) 228 { 229 return rq->rq_disk && 230 (rq->cmd_flags & REQ_IO_STAT) && 231 (rq->cmd_type == REQ_TYPE_FS); 232 } 233 234 /* 235 * Internal io_context interface 236 */ 237 void get_io_context(struct io_context *ioc); 238 struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q); 239 struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q, 240 gfp_t gfp_mask); 241 void ioc_clear_queue(struct request_queue *q); 242 243 int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node); 244 245 /** 246 * create_io_context - try to create task->io_context 247 * @gfp_mask: allocation mask 248 * @node: allocation node 249 * 250 * If %current->io_context is %NULL, allocate a new io_context and install 251 * it. Returns the current %current->io_context which may be %NULL if 252 * allocation failed. 253 * 254 * Note that this function can't be called with IRQ disabled because 255 * task_lock which protects %current->io_context is IRQ-unsafe. 256 */ 257 static inline struct io_context *create_io_context(gfp_t gfp_mask, int node) 258 { 259 WARN_ON_ONCE(irqs_disabled()); 260 if (unlikely(!current->io_context)) 261 create_task_io_context(current, gfp_mask, node); 262 return current->io_context; 263 } 264 265 /* 266 * Internal throttling interface 267 */ 268 #ifdef CONFIG_BLK_DEV_THROTTLING 269 extern bool blk_throtl_bio(struct request_queue *q, struct bio *bio); 270 extern void blk_throtl_drain(struct request_queue *q); 271 extern int blk_throtl_init(struct request_queue *q); 272 extern void blk_throtl_exit(struct request_queue *q); 273 #else /* CONFIG_BLK_DEV_THROTTLING */ 274 static inline bool blk_throtl_bio(struct request_queue *q, struct bio *bio) 275 { 276 return false; 277 } 278 static inline void blk_throtl_drain(struct request_queue *q) { } 279 static inline int blk_throtl_init(struct request_queue *q) { return 0; } 280 static inline void blk_throtl_exit(struct request_queue *q) { } 281 #endif /* CONFIG_BLK_DEV_THROTTLING */ 282 283 #endif /* BLK_INTERNAL_H */ 284