1 #ifndef INT_BLK_MQ_H 2 #define INT_BLK_MQ_H 3 4 #include "blk-stat.h" 5 6 struct blk_mq_tag_set; 7 8 struct blk_mq_ctx { 9 struct { 10 spinlock_t lock; 11 struct list_head rq_list; 12 } ____cacheline_aligned_in_smp; 13 14 unsigned int cpu; 15 unsigned int index_hw; 16 17 /* incremented at dispatch time */ 18 unsigned long rq_dispatched[2]; 19 unsigned long rq_merged; 20 21 /* incremented at completion time */ 22 unsigned long ____cacheline_aligned_in_smp rq_completed[2]; 23 struct blk_rq_stat stat[2]; 24 25 struct request_queue *queue; 26 struct kobject kobj; 27 } ____cacheline_aligned_in_smp; 28 29 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); 30 void blk_mq_freeze_queue(struct request_queue *q); 31 void blk_mq_free_queue(struct request_queue *q); 32 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr); 33 void blk_mq_wake_waiters(struct request_queue *q); 34 bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *, struct list_head *); 35 36 /* 37 * CPU hotplug helpers 38 */ 39 void blk_mq_enable_hotplug(void); 40 void blk_mq_disable_hotplug(void); 41 42 /* 43 * CPU -> queue mappings 44 */ 45 extern int blk_mq_hw_queue_to_node(unsigned int *map, unsigned int); 46 47 static inline struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, 48 int cpu) 49 { 50 return q->queue_hw_ctx[q->mq_map[cpu]]; 51 } 52 53 /* 54 * sysfs helpers 55 */ 56 extern int blk_mq_sysfs_register(struct request_queue *q); 57 extern void blk_mq_sysfs_unregister(struct request_queue *q); 58 extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx); 59 60 extern void blk_mq_rq_timed_out(struct request *req, bool reserved); 61 62 void blk_mq_release(struct request_queue *q); 63 64 static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q, 65 unsigned int cpu) 66 { 67 return per_cpu_ptr(q->queue_ctx, cpu); 68 } 69 70 /* 71 * This assumes per-cpu software queueing queues. They could be per-node 72 * as well, for instance. For now this is hardcoded as-is. Note that we don't 73 * care about preemption, since we know the ctx's are persistent. This does 74 * mean that we can't rely on ctx always matching the currently running CPU. 75 */ 76 static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q) 77 { 78 return __blk_mq_get_ctx(q, get_cpu()); 79 } 80 81 static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx) 82 { 83 put_cpu(); 84 } 85 86 struct blk_mq_alloc_data { 87 /* input parameter */ 88 struct request_queue *q; 89 unsigned int flags; 90 91 /* input & output parameter */ 92 struct blk_mq_ctx *ctx; 93 struct blk_mq_hw_ctx *hctx; 94 }; 95 96 static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data, 97 struct request_queue *q, unsigned int flags, 98 struct blk_mq_ctx *ctx, struct blk_mq_hw_ctx *hctx) 99 { 100 data->q = q; 101 data->flags = flags; 102 data->ctx = ctx; 103 data->hctx = hctx; 104 } 105 106 static inline bool blk_mq_hctx_stopped(struct blk_mq_hw_ctx *hctx) 107 { 108 return test_bit(BLK_MQ_S_STOPPED, &hctx->state); 109 } 110 111 static inline bool blk_mq_hw_queue_mapped(struct blk_mq_hw_ctx *hctx) 112 { 113 return hctx->nr_ctx && hctx->tags; 114 } 115 116 #endif 117