1 /* 2 * Functions to sequence FLUSH and FUA writes. 3 */ 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/bio.h> 7 #include <linux/blkdev.h> 8 #include <linux/gfp.h> 9 10 #include "blk.h" 11 12 /* FLUSH/FUA sequences */ 13 enum { 14 QUEUE_FSEQ_STARTED = (1 << 0), /* flushing in progress */ 15 QUEUE_FSEQ_PREFLUSH = (1 << 1), /* pre-flushing in progress */ 16 QUEUE_FSEQ_DATA = (1 << 2), /* data write in progress */ 17 QUEUE_FSEQ_POSTFLUSH = (1 << 3), /* post-flushing in progress */ 18 QUEUE_FSEQ_DONE = (1 << 4), 19 }; 20 21 static struct request *queue_next_fseq(struct request_queue *q); 22 23 unsigned blk_flush_cur_seq(struct request_queue *q) 24 { 25 if (!q->flush_seq) 26 return 0; 27 return 1 << ffz(q->flush_seq); 28 } 29 30 static struct request *blk_flush_complete_seq(struct request_queue *q, 31 unsigned seq, int error) 32 { 33 struct request *next_rq = NULL; 34 35 if (error && !q->flush_err) 36 q->flush_err = error; 37 38 BUG_ON(q->flush_seq & seq); 39 q->flush_seq |= seq; 40 41 if (blk_flush_cur_seq(q) != QUEUE_FSEQ_DONE) { 42 /* not complete yet, queue the next flush sequence */ 43 next_rq = queue_next_fseq(q); 44 } else { 45 /* complete this flush request */ 46 __blk_end_request_all(q->orig_flush_rq, q->flush_err); 47 q->orig_flush_rq = NULL; 48 q->flush_seq = 0; 49 50 /* dispatch the next flush if there's one */ 51 if (!list_empty(&q->pending_flushes)) { 52 next_rq = list_entry_rq(q->pending_flushes.next); 53 list_move(&next_rq->queuelist, &q->queue_head); 54 } 55 } 56 return next_rq; 57 } 58 59 static void blk_flush_complete_seq_end_io(struct request_queue *q, 60 unsigned seq, int error) 61 { 62 bool was_empty = elv_queue_empty(q); 63 struct request *next_rq; 64 65 next_rq = blk_flush_complete_seq(q, seq, error); 66 67 /* 68 * Moving a request silently to empty queue_head may stall the 69 * queue. Kick the queue in those cases. 70 */ 71 if (was_empty && next_rq) 72 __blk_run_queue(q); 73 } 74 75 static void pre_flush_end_io(struct request *rq, int error) 76 { 77 elv_completed_request(rq->q, rq); 78 blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_PREFLUSH, error); 79 } 80 81 static void flush_data_end_io(struct request *rq, int error) 82 { 83 elv_completed_request(rq->q, rq); 84 blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_DATA, error); 85 } 86 87 static void post_flush_end_io(struct request *rq, int error) 88 { 89 elv_completed_request(rq->q, rq); 90 blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_POSTFLUSH, error); 91 } 92 93 static void init_flush_request(struct request *rq, struct gendisk *disk) 94 { 95 rq->cmd_type = REQ_TYPE_FS; 96 rq->cmd_flags = WRITE_FLUSH; 97 rq->rq_disk = disk; 98 } 99 100 static struct request *queue_next_fseq(struct request_queue *q) 101 { 102 struct request *orig_rq = q->orig_flush_rq; 103 struct request *rq = &q->flush_rq; 104 105 blk_rq_init(q, rq); 106 107 switch (blk_flush_cur_seq(q)) { 108 case QUEUE_FSEQ_PREFLUSH: 109 init_flush_request(rq, orig_rq->rq_disk); 110 rq->end_io = pre_flush_end_io; 111 break; 112 case QUEUE_FSEQ_DATA: 113 init_request_from_bio(rq, orig_rq->bio); 114 /* 115 * orig_rq->rq_disk may be different from 116 * bio->bi_bdev->bd_disk if orig_rq got here through 117 * remapping drivers. Make sure rq->rq_disk points 118 * to the same one as orig_rq. 119 */ 120 rq->rq_disk = orig_rq->rq_disk; 121 rq->cmd_flags &= ~(REQ_FLUSH | REQ_FUA); 122 rq->cmd_flags |= orig_rq->cmd_flags & (REQ_FLUSH | REQ_FUA); 123 rq->end_io = flush_data_end_io; 124 break; 125 case QUEUE_FSEQ_POSTFLUSH: 126 init_flush_request(rq, orig_rq->rq_disk); 127 rq->end_io = post_flush_end_io; 128 break; 129 default: 130 BUG(); 131 } 132 133 elv_insert(q, rq, ELEVATOR_INSERT_FRONT); 134 return rq; 135 } 136 137 struct request *blk_do_flush(struct request_queue *q, struct request *rq) 138 { 139 unsigned int fflags = q->flush_flags; /* may change, cache it */ 140 bool has_flush = fflags & REQ_FLUSH, has_fua = fflags & REQ_FUA; 141 bool do_preflush = has_flush && (rq->cmd_flags & REQ_FLUSH); 142 bool do_postflush = has_flush && !has_fua && (rq->cmd_flags & REQ_FUA); 143 unsigned skip = 0; 144 145 /* 146 * Special case. If there's data but flush is not necessary, 147 * the request can be issued directly. 148 * 149 * Flush w/o data should be able to be issued directly too but 150 * currently some drivers assume that rq->bio contains 151 * non-zero data if it isn't NULL and empty FLUSH requests 152 * getting here usually have bio's without data. 153 */ 154 if (blk_rq_sectors(rq) && !do_preflush && !do_postflush) { 155 rq->cmd_flags &= ~REQ_FLUSH; 156 if (!has_fua) 157 rq->cmd_flags &= ~REQ_FUA; 158 return rq; 159 } 160 161 /* 162 * Sequenced flushes can't be processed in parallel. If 163 * another one is already in progress, queue for later 164 * processing. 165 */ 166 if (q->flush_seq) { 167 list_move_tail(&rq->queuelist, &q->pending_flushes); 168 return NULL; 169 } 170 171 /* 172 * Start a new flush sequence 173 */ 174 q->flush_err = 0; 175 q->flush_seq |= QUEUE_FSEQ_STARTED; 176 177 /* adjust FLUSH/FUA of the original request and stash it away */ 178 rq->cmd_flags &= ~REQ_FLUSH; 179 if (!has_fua) 180 rq->cmd_flags &= ~REQ_FUA; 181 blk_dequeue_request(rq); 182 q->orig_flush_rq = rq; 183 184 /* skip unneded sequences and return the first one */ 185 if (!do_preflush) 186 skip |= QUEUE_FSEQ_PREFLUSH; 187 if (!blk_rq_sectors(rq)) 188 skip |= QUEUE_FSEQ_DATA; 189 if (!do_postflush) 190 skip |= QUEUE_FSEQ_POSTFLUSH; 191 return blk_flush_complete_seq(q, skip, 0); 192 } 193 194 static void bio_end_flush(struct bio *bio, int err) 195 { 196 if (err) 197 clear_bit(BIO_UPTODATE, &bio->bi_flags); 198 if (bio->bi_private) 199 complete(bio->bi_private); 200 bio_put(bio); 201 } 202 203 /** 204 * blkdev_issue_flush - queue a flush 205 * @bdev: blockdev to issue flush for 206 * @gfp_mask: memory allocation flags (for bio_alloc) 207 * @error_sector: error sector 208 * 209 * Description: 210 * Issue a flush for the block device in question. Caller can supply 211 * room for storing the error offset in case of a flush error, if they 212 * wish to. If WAIT flag is not passed then caller may check only what 213 * request was pushed in some internal queue for later handling. 214 */ 215 int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, 216 sector_t *error_sector) 217 { 218 DECLARE_COMPLETION_ONSTACK(wait); 219 struct request_queue *q; 220 struct bio *bio; 221 int ret = 0; 222 223 if (bdev->bd_disk == NULL) 224 return -ENXIO; 225 226 q = bdev_get_queue(bdev); 227 if (!q) 228 return -ENXIO; 229 230 /* 231 * some block devices may not have their queue correctly set up here 232 * (e.g. loop device without a backing file) and so issuing a flush 233 * here will panic. Ensure there is a request function before issuing 234 * the flush. 235 */ 236 if (!q->make_request_fn) 237 return -ENXIO; 238 239 bio = bio_alloc(gfp_mask, 0); 240 bio->bi_end_io = bio_end_flush; 241 bio->bi_bdev = bdev; 242 bio->bi_private = &wait; 243 244 bio_get(bio); 245 submit_bio(WRITE_FLUSH, bio); 246 wait_for_completion(&wait); 247 248 /* 249 * The driver must store the error location in ->bi_sector, if 250 * it supports it. For non-stacked drivers, this should be 251 * copied from blk_rq_pos(rq). 252 */ 253 if (error_sector) 254 *error_sector = bio->bi_sector; 255 256 if (!bio_flagged(bio, BIO_UPTODATE)) 257 ret = -EIO; 258 259 bio_put(bio); 260 return ret; 261 } 262 EXPORT_SYMBOL(blkdev_issue_flush); 263