18839a0e0STejun Heo /* 24fed947cSTejun Heo * Functions to sequence FLUSH and FUA writes. 3ae1b1539STejun Heo * 4ae1b1539STejun Heo * Copyright (C) 2011 Max Planck Institute for Gravitational Physics 5ae1b1539STejun Heo * Copyright (C) 2011 Tejun Heo <tj@kernel.org> 6ae1b1539STejun Heo * 7ae1b1539STejun Heo * This file is released under the GPLv2. 8ae1b1539STejun Heo * 9ae1b1539STejun Heo * REQ_{FLUSH|FUA} requests are decomposed to sequences consisted of three 10ae1b1539STejun Heo * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request 11ae1b1539STejun Heo * properties and hardware capability. 12ae1b1539STejun Heo * 1328a8f0d3SMike Christie * If a request doesn't have data, only REQ_PREFLUSH makes sense, which 1428a8f0d3SMike Christie * indicates a simple flush request. If there is data, REQ_PREFLUSH indicates 15ae1b1539STejun Heo * that the device cache should be flushed before the data is executed, and 16ae1b1539STejun Heo * REQ_FUA means that the data must be on non-volatile media on request 17ae1b1539STejun Heo * completion. 18ae1b1539STejun Heo * 19ae1b1539STejun Heo * If the device doesn't have writeback cache, FLUSH and FUA don't make any 20ae1b1539STejun Heo * difference. The requests are either completed immediately if there's no 21ae1b1539STejun Heo * data or executed as normal requests otherwise. 22ae1b1539STejun Heo * 2328a8f0d3SMike Christie * If the device has writeback cache and supports FUA, REQ_PREFLUSH is 24ae1b1539STejun Heo * translated to PREFLUSH but REQ_FUA is passed down directly with DATA. 25ae1b1539STejun Heo * 2628a8f0d3SMike Christie * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH 2728a8f0d3SMike Christie * is translated to PREFLUSH and REQ_FUA to POSTFLUSH. 28ae1b1539STejun Heo * 29ae1b1539STejun Heo * The actual execution of flush is double buffered. Whenever a request 30ae1b1539STejun Heo * needs to execute PRE or POSTFLUSH, it queues at 317c94e1c1SMing Lei * fq->flush_queue[fq->flush_pending_idx]. Once certain criteria are met, a 323a5e02ceSMike Christie * REQ_OP_FLUSH is issued and the pending_idx is toggled. When the flush 33ae1b1539STejun Heo * completes, all the requests which were pending are proceeded to the next 34ae1b1539STejun Heo * step. This allows arbitrary merging of different types of FLUSH/FUA 35ae1b1539STejun Heo * requests. 36ae1b1539STejun Heo * 37ae1b1539STejun Heo * Currently, the following conditions are used to determine when to issue 38ae1b1539STejun Heo * flush. 39ae1b1539STejun Heo * 40ae1b1539STejun Heo * C1. At any given time, only one flush shall be in progress. This makes 41ae1b1539STejun Heo * double buffering sufficient. 42ae1b1539STejun Heo * 43ae1b1539STejun Heo * C2. Flush is deferred if any request is executing DATA of its sequence. 44ae1b1539STejun Heo * This avoids issuing separate POSTFLUSHes for requests which shared 45ae1b1539STejun Heo * PREFLUSH. 46ae1b1539STejun Heo * 47ae1b1539STejun Heo * C3. The second condition is ignored if there is a request which has 48ae1b1539STejun Heo * waited longer than FLUSH_PENDING_TIMEOUT. This is to avoid 49ae1b1539STejun Heo * starvation in the unlikely case where there are continuous stream of 50ae1b1539STejun Heo * FUA (without FLUSH) requests. 51ae1b1539STejun Heo * 52ae1b1539STejun Heo * For devices which support FUA, it isn't clear whether C2 (and thus C3) 53ae1b1539STejun Heo * is beneficial. 54ae1b1539STejun Heo * 55ae1b1539STejun Heo * Note that a sequenced FLUSH/FUA request with DATA is completed twice. 56ae1b1539STejun Heo * Once while executing DATA and again after the whole sequence is 57ae1b1539STejun Heo * complete. The first completion updates the contained bio but doesn't 58ae1b1539STejun Heo * finish it so that the bio submitter is notified only after the whole 59e8064021SChristoph Hellwig * sequence is complete. This is implemented by testing RQF_FLUSH_SEQ in 60ae1b1539STejun Heo * req_bio_endio(). 61ae1b1539STejun Heo * 62ae1b1539STejun Heo * The above peculiarity requires that each FLUSH/FUA request has only one 63ae1b1539STejun Heo * bio attached to it, which is guaranteed as they aren't allowed to be 64ae1b1539STejun Heo * merged in the usual way. 658839a0e0STejun Heo */ 66ae1b1539STejun Heo 678839a0e0STejun Heo #include <linux/kernel.h> 688839a0e0STejun Heo #include <linux/module.h> 698839a0e0STejun Heo #include <linux/bio.h> 708839a0e0STejun Heo #include <linux/blkdev.h> 718839a0e0STejun Heo #include <linux/gfp.h> 72320ae51fSJens Axboe #include <linux/blk-mq.h> 738839a0e0STejun Heo 748839a0e0STejun Heo #include "blk.h" 75320ae51fSJens Axboe #include "blk-mq.h" 760048b483SMing Lei #include "blk-mq-tag.h" 77bd166ef1SJens Axboe #include "blk-mq-sched.h" 788839a0e0STejun Heo 794fed947cSTejun Heo /* FLUSH/FUA sequences */ 804fed947cSTejun Heo enum { 81ae1b1539STejun Heo REQ_FSEQ_PREFLUSH = (1 << 0), /* pre-flushing in progress */ 82ae1b1539STejun Heo REQ_FSEQ_DATA = (1 << 1), /* data write in progress */ 83ae1b1539STejun Heo REQ_FSEQ_POSTFLUSH = (1 << 2), /* post-flushing in progress */ 84ae1b1539STejun Heo REQ_FSEQ_DONE = (1 << 3), 85ae1b1539STejun Heo 86ae1b1539STejun Heo REQ_FSEQ_ACTIONS = REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA | 87ae1b1539STejun Heo REQ_FSEQ_POSTFLUSH, 88ae1b1539STejun Heo 89ae1b1539STejun Heo /* 90ae1b1539STejun Heo * If flush has been pending longer than the following timeout, 91ae1b1539STejun Heo * it's issued even if flush_data requests are still in flight. 92ae1b1539STejun Heo */ 93ae1b1539STejun Heo FLUSH_PENDING_TIMEOUT = 5 * HZ, 944fed947cSTejun Heo }; 954fed947cSTejun Heo 960bae352dSMing Lei static bool blk_kick_flush(struct request_queue *q, 970bae352dSMing Lei struct blk_flush_queue *fq); 988839a0e0STejun Heo 99c888a8f9SJens Axboe static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq) 1008839a0e0STejun Heo { 101ae1b1539STejun Heo unsigned int policy = 0; 102ae1b1539STejun Heo 103fa1bf42fSJeff Moyer if (blk_rq_sectors(rq)) 104fa1bf42fSJeff Moyer policy |= REQ_FSEQ_DATA; 105fa1bf42fSJeff Moyer 106c888a8f9SJens Axboe if (fflags & (1UL << QUEUE_FLAG_WC)) { 10728a8f0d3SMike Christie if (rq->cmd_flags & REQ_PREFLUSH) 108ae1b1539STejun Heo policy |= REQ_FSEQ_PREFLUSH; 109c888a8f9SJens Axboe if (!(fflags & (1UL << QUEUE_FLAG_FUA)) && 110c888a8f9SJens Axboe (rq->cmd_flags & REQ_FUA)) 111ae1b1539STejun Heo policy |= REQ_FSEQ_POSTFLUSH; 112ae1b1539STejun Heo } 113ae1b1539STejun Heo return policy; 1148839a0e0STejun Heo } 1158839a0e0STejun Heo 116ae1b1539STejun Heo static unsigned int blk_flush_cur_seq(struct request *rq) 1178839a0e0STejun Heo { 118ae1b1539STejun Heo return 1 << ffz(rq->flush.seq); 1198839a0e0STejun Heo } 1208839a0e0STejun Heo 121ae1b1539STejun Heo static void blk_flush_restore_request(struct request *rq) 12247f70d5aSTejun Heo { 12347f70d5aSTejun Heo /* 124ae1b1539STejun Heo * After flush data completion, @rq->bio is %NULL but we need to 125ae1b1539STejun Heo * complete the bio again. @rq->biotail is guaranteed to equal the 126ae1b1539STejun Heo * original @rq->bio. Restore it. 12747f70d5aSTejun Heo */ 128ae1b1539STejun Heo rq->bio = rq->biotail; 129ae1b1539STejun Heo 130ae1b1539STejun Heo /* make @rq a normal request */ 131e8064021SChristoph Hellwig rq->rq_flags &= ~RQF_FLUSH_SEQ; 1324853abaaSJeff Moyer rq->end_io = rq->flush.saved_end_io; 133320ae51fSJens Axboe } 134320ae51fSJens Axboe 13510beafc1SMike Snitzer static bool blk_flush_queue_rq(struct request *rq, bool add_front) 136320ae51fSJens Axboe { 13718741986SChristoph Hellwig if (rq->q->mq_ops) { 1382b053acaSBart Van Assche blk_mq_add_to_requeue_list(rq, add_front, true); 13918741986SChristoph Hellwig return false; 14018741986SChristoph Hellwig } else { 14110beafc1SMike Snitzer if (add_front) 14210beafc1SMike Snitzer list_add(&rq->queuelist, &rq->q->queue_head); 14310beafc1SMike Snitzer else 14418741986SChristoph Hellwig list_add_tail(&rq->queuelist, &rq->q->queue_head); 14518741986SChristoph Hellwig return true; 14618741986SChristoph Hellwig } 14747f70d5aSTejun Heo } 14847f70d5aSTejun Heo 149ae1b1539STejun Heo /** 150ae1b1539STejun Heo * blk_flush_complete_seq - complete flush sequence 151ae1b1539STejun Heo * @rq: FLUSH/FUA request being sequenced 1520bae352dSMing Lei * @fq: flush queue 153ae1b1539STejun Heo * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero) 154ae1b1539STejun Heo * @error: whether an error occurred 155ae1b1539STejun Heo * 156ae1b1539STejun Heo * @rq just completed @seq part of its flush sequence, record the 157ae1b1539STejun Heo * completion and trigger the next step. 158ae1b1539STejun Heo * 159ae1b1539STejun Heo * CONTEXT: 1607c94e1c1SMing Lei * spin_lock_irq(q->queue_lock or fq->mq_flush_lock) 161ae1b1539STejun Heo * 162ae1b1539STejun Heo * RETURNS: 163ae1b1539STejun Heo * %true if requests were added to the dispatch queue, %false otherwise. 164ae1b1539STejun Heo */ 1650bae352dSMing Lei static bool blk_flush_complete_seq(struct request *rq, 1660bae352dSMing Lei struct blk_flush_queue *fq, 1670bae352dSMing Lei unsigned int seq, int error) 1688839a0e0STejun Heo { 169ae1b1539STejun Heo struct request_queue *q = rq->q; 1707c94e1c1SMing Lei struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; 171320ae51fSJens Axboe bool queued = false, kicked; 1728839a0e0STejun Heo 173ae1b1539STejun Heo BUG_ON(rq->flush.seq & seq); 174ae1b1539STejun Heo rq->flush.seq |= seq; 1758839a0e0STejun Heo 176ae1b1539STejun Heo if (likely(!error)) 177ae1b1539STejun Heo seq = blk_flush_cur_seq(rq); 178ae1b1539STejun Heo else 179ae1b1539STejun Heo seq = REQ_FSEQ_DONE; 1808839a0e0STejun Heo 181ae1b1539STejun Heo switch (seq) { 182ae1b1539STejun Heo case REQ_FSEQ_PREFLUSH: 183ae1b1539STejun Heo case REQ_FSEQ_POSTFLUSH: 184ae1b1539STejun Heo /* queue for flush */ 185ae1b1539STejun Heo if (list_empty(pending)) 1867c94e1c1SMing Lei fq->flush_pending_since = jiffies; 187ae1b1539STejun Heo list_move_tail(&rq->flush.list, pending); 1888839a0e0STejun Heo break; 189ae1b1539STejun Heo 190ae1b1539STejun Heo case REQ_FSEQ_DATA: 1917c94e1c1SMing Lei list_move_tail(&rq->flush.list, &fq->flush_data_in_flight); 19210beafc1SMike Snitzer queued = blk_flush_queue_rq(rq, true); 193ae1b1539STejun Heo break; 194ae1b1539STejun Heo 195ae1b1539STejun Heo case REQ_FSEQ_DONE: 19609d60c70STejun Heo /* 197ae1b1539STejun Heo * @rq was previously adjusted by blk_flush_issue() for 198ae1b1539STejun Heo * flush sequencing and may already have gone through the 199ae1b1539STejun Heo * flush data request completion path. Restore @rq for 200ae1b1539STejun Heo * normal completion and end it. 20109d60c70STejun Heo */ 202ae1b1539STejun Heo BUG_ON(!list_empty(&rq->queuelist)); 203ae1b1539STejun Heo list_del_init(&rq->flush.list); 204ae1b1539STejun Heo blk_flush_restore_request(rq); 205320ae51fSJens Axboe if (q->mq_ops) 206c8a446adSChristoph Hellwig blk_mq_end_request(rq, error); 207320ae51fSJens Axboe else 208ae1b1539STejun Heo __blk_end_request_all(rq, error); 2098839a0e0STejun Heo break; 210ae1b1539STejun Heo 2118839a0e0STejun Heo default: 2128839a0e0STejun Heo BUG(); 2138839a0e0STejun Heo } 214cde4c406SChristoph Hellwig 2150bae352dSMing Lei kicked = blk_kick_flush(q, fq); 216320ae51fSJens Axboe return kicked | queued; 2178839a0e0STejun Heo } 2188839a0e0STejun Heo 219ae1b1539STejun Heo static void flush_end_io(struct request *flush_rq, int error) 2208839a0e0STejun Heo { 221ae1b1539STejun Heo struct request_queue *q = flush_rq->q; 222320ae51fSJens Axboe struct list_head *running; 223ae1b1539STejun Heo bool queued = false; 224ae1b1539STejun Heo struct request *rq, *n; 225320ae51fSJens Axboe unsigned long flags = 0; 226e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx); 2278839a0e0STejun Heo 22822302375SShaohua Li if (q->mq_ops) { 2290048b483SMing Lei struct blk_mq_hw_ctx *hctx; 2300048b483SMing Lei 2310048b483SMing Lei /* release the tag's ownership to the req cloned from */ 2327c94e1c1SMing Lei spin_lock_irqsave(&fq->mq_flush_lock, flags); 2337d7e0f90SChristoph Hellwig hctx = blk_mq_map_queue(q, flush_rq->mq_ctx->cpu); 2340048b483SMing Lei blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq); 2357ddab5deSMing Lei flush_rq->tag = -1; 23622302375SShaohua Li } 23718741986SChristoph Hellwig 2387c94e1c1SMing Lei running = &fq->flush_queue[fq->flush_running_idx]; 2397c94e1c1SMing Lei BUG_ON(fq->flush_pending_idx == fq->flush_running_idx); 240ae1b1539STejun Heo 241ae1b1539STejun Heo /* account completion of the flush request */ 2427c94e1c1SMing Lei fq->flush_running_idx ^= 1; 243320ae51fSJens Axboe 244320ae51fSJens Axboe if (!q->mq_ops) 245ae1b1539STejun Heo elv_completed_request(q, flush_rq); 246ae1b1539STejun Heo 247ae1b1539STejun Heo /* and push the waiting requests to the next stage */ 248ae1b1539STejun Heo list_for_each_entry_safe(rq, n, running, flush.list) { 249ae1b1539STejun Heo unsigned int seq = blk_flush_cur_seq(rq); 250ae1b1539STejun Heo 251ae1b1539STejun Heo BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH); 2520bae352dSMing Lei queued |= blk_flush_complete_seq(rq, fq, seq, error); 253ae1b1539STejun Heo } 254ae1b1539STejun Heo 2558839a0e0STejun Heo /* 2563ac0cc45Sshaohua.li@intel.com * Kick the queue to avoid stall for two cases: 2573ac0cc45Sshaohua.li@intel.com * 1. Moving a request silently to empty queue_head may stall the 2583ac0cc45Sshaohua.li@intel.com * queue. 2593ac0cc45Sshaohua.li@intel.com * 2. When flush request is running in non-queueable queue, the 2603ac0cc45Sshaohua.li@intel.com * queue is hold. Restart the queue after flush request is finished 2613ac0cc45Sshaohua.li@intel.com * to avoid stall. 2623ac0cc45Sshaohua.li@intel.com * This function is called from request completion path and calling 2633ac0cc45Sshaohua.li@intel.com * directly into request_fn may confuse the driver. Always use 2643ac0cc45Sshaohua.li@intel.com * kblockd. 2658839a0e0STejun Heo */ 2667c94e1c1SMing Lei if (queued || fq->flush_queue_delayed) { 26718741986SChristoph Hellwig WARN_ON(q->mq_ops); 26824ecfbe2SChristoph Hellwig blk_run_queue_async(q); 269320ae51fSJens Axboe } 2707c94e1c1SMing Lei fq->flush_queue_delayed = 0; 271320ae51fSJens Axboe if (q->mq_ops) 2727c94e1c1SMing Lei spin_unlock_irqrestore(&fq->mq_flush_lock, flags); 273320ae51fSJens Axboe } 274320ae51fSJens Axboe 275ae1b1539STejun Heo /** 276ae1b1539STejun Heo * blk_kick_flush - consider issuing flush request 277ae1b1539STejun Heo * @q: request_queue being kicked 2780bae352dSMing Lei * @fq: flush queue 2794fed947cSTejun Heo * 280ae1b1539STejun Heo * Flush related states of @q have changed, consider issuing flush request. 281ae1b1539STejun Heo * Please read the comment at the top of this file for more info. 282ae1b1539STejun Heo * 283ae1b1539STejun Heo * CONTEXT: 2847c94e1c1SMing Lei * spin_lock_irq(q->queue_lock or fq->mq_flush_lock) 285ae1b1539STejun Heo * 286ae1b1539STejun Heo * RETURNS: 287ae1b1539STejun Heo * %true if flush was issued, %false otherwise. 2888839a0e0STejun Heo */ 2890bae352dSMing Lei static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) 290ae1b1539STejun Heo { 2917c94e1c1SMing Lei struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; 292ae1b1539STejun Heo struct request *first_rq = 293ae1b1539STejun Heo list_first_entry(pending, struct request, flush.list); 2947c94e1c1SMing Lei struct request *flush_rq = fq->flush_rq; 295ae1b1539STejun Heo 296ae1b1539STejun Heo /* C1 described at the top of this file */ 2977c94e1c1SMing Lei if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending)) 298ae1b1539STejun Heo return false; 299ae1b1539STejun Heo 300ae1b1539STejun Heo /* C2 and C3 */ 3017c94e1c1SMing Lei if (!list_empty(&fq->flush_data_in_flight) && 302ae1b1539STejun Heo time_before(jiffies, 3037c94e1c1SMing Lei fq->flush_pending_since + FLUSH_PENDING_TIMEOUT)) 304ae1b1539STejun Heo return false; 305ae1b1539STejun Heo 306ae1b1539STejun Heo /* 307ae1b1539STejun Heo * Issue flush and toggle pending_idx. This makes pending_idx 308ae1b1539STejun Heo * different from running_idx, which means flush is in flight. 309ae1b1539STejun Heo */ 3107c94e1c1SMing Lei fq->flush_pending_idx ^= 1; 31118741986SChristoph Hellwig 3127ddab5deSMing Lei blk_rq_init(q, flush_rq); 313f70ced09SMing Lei 314f70ced09SMing Lei /* 315f70ced09SMing Lei * Borrow tag from the first request since they can't 3160048b483SMing Lei * be in flight at the same time. And acquire the tag's 3170048b483SMing Lei * ownership for flush req. 318f70ced09SMing Lei */ 319f70ced09SMing Lei if (q->mq_ops) { 3200048b483SMing Lei struct blk_mq_hw_ctx *hctx; 3210048b483SMing Lei 322f70ced09SMing Lei flush_rq->mq_ctx = first_rq->mq_ctx; 323f70ced09SMing Lei flush_rq->tag = first_rq->tag; 3240048b483SMing Lei fq->orig_rq = first_rq; 3250048b483SMing Lei 3267d7e0f90SChristoph Hellwig hctx = blk_mq_map_queue(q, first_rq->mq_ctx->cpu); 3270048b483SMing Lei blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq); 328f70ced09SMing Lei } 329320ae51fSJens Axboe 3307ddab5deSMing Lei flush_rq->cmd_type = REQ_TYPE_FS; 33170fd7614SChristoph Hellwig flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH; 332e8064021SChristoph Hellwig flush_rq->rq_flags |= RQF_FLUSH_SEQ; 3337ddab5deSMing Lei flush_rq->rq_disk = first_rq->rq_disk; 3347ddab5deSMing Lei flush_rq->end_io = flush_end_io; 335ae1b1539STejun Heo 3367ddab5deSMing Lei return blk_flush_queue_rq(flush_rq, false); 337ae1b1539STejun Heo } 338ae1b1539STejun Heo 339ae1b1539STejun Heo static void flush_data_end_io(struct request *rq, int error) 340ae1b1539STejun Heo { 341ae1b1539STejun Heo struct request_queue *q = rq->q; 342e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); 343ae1b1539STejun Heo 3448839a0e0STejun Heo /* 34594d7dea4SMing Lei * Updating q->in_flight[] here for making this tag usable 34694d7dea4SMing Lei * early. Because in blk_queue_start_tag(), 34794d7dea4SMing Lei * q->in_flight[BLK_RW_ASYNC] is used to limit async I/O and 34894d7dea4SMing Lei * reserve tags for sync I/O. 34994d7dea4SMing Lei * 35094d7dea4SMing Lei * More importantly this way can avoid the following I/O 35194d7dea4SMing Lei * deadlock: 35294d7dea4SMing Lei * 35394d7dea4SMing Lei * - suppose there are 40 fua requests comming to flush queue 35494d7dea4SMing Lei * and queue depth is 31 35594d7dea4SMing Lei * - 30 rqs are scheduled then blk_queue_start_tag() can't alloc 35694d7dea4SMing Lei * tag for async I/O any more 35794d7dea4SMing Lei * - all the 30 rqs are completed before FLUSH_PENDING_TIMEOUT 35894d7dea4SMing Lei * and flush_data_end_io() is called 35994d7dea4SMing Lei * - the other rqs still can't go ahead if not updating 36094d7dea4SMing Lei * q->in_flight[BLK_RW_ASYNC] here, meantime these rqs 36194d7dea4SMing Lei * are held in flush data queue and make no progress of 36294d7dea4SMing Lei * handling post flush rq 36394d7dea4SMing Lei * - only after the post flush rq is handled, all these rqs 36494d7dea4SMing Lei * can be completed 36594d7dea4SMing Lei */ 36694d7dea4SMing Lei 36794d7dea4SMing Lei elv_completed_request(q, rq); 36894d7dea4SMing Lei 36994d7dea4SMing Lei /* for avoiding double accounting */ 37036869cb9SLinus Torvalds rq->rq_flags &= ~RQF_STARTED; 37194d7dea4SMing Lei 37294d7dea4SMing Lei /* 373e83a46bbSTejun Heo * After populating an empty queue, kick it to avoid stall. Read 374e83a46bbSTejun Heo * the comment in flush_end_io(). 3758839a0e0STejun Heo */ 3760bae352dSMing Lei if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error)) 37724ecfbe2SChristoph Hellwig blk_run_queue_async(q); 378ae1b1539STejun Heo } 379ae1b1539STejun Heo 380320ae51fSJens Axboe static void mq_flush_data_end_io(struct request *rq, int error) 381320ae51fSJens Axboe { 382320ae51fSJens Axboe struct request_queue *q = rq->q; 383320ae51fSJens Axboe struct blk_mq_hw_ctx *hctx; 384e97c293cSMing Lei struct blk_mq_ctx *ctx = rq->mq_ctx; 385320ae51fSJens Axboe unsigned long flags; 386e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx); 387320ae51fSJens Axboe 3887d7e0f90SChristoph Hellwig hctx = blk_mq_map_queue(q, ctx->cpu); 389320ae51fSJens Axboe 390320ae51fSJens Axboe /* 391320ae51fSJens Axboe * After populating an empty queue, kick it to avoid stall. Read 392320ae51fSJens Axboe * the comment in flush_end_io(). 393320ae51fSJens Axboe */ 3947c94e1c1SMing Lei spin_lock_irqsave(&fq->mq_flush_lock, flags); 395bd166ef1SJens Axboe blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error); 3967c94e1c1SMing Lei spin_unlock_irqrestore(&fq->mq_flush_lock, flags); 397bd166ef1SJens Axboe 398bd166ef1SJens Axboe blk_mq_run_hw_queue(hctx, true); 399320ae51fSJens Axboe } 400320ae51fSJens Axboe 401ae1b1539STejun Heo /** 402ae1b1539STejun Heo * blk_insert_flush - insert a new FLUSH/FUA request 403ae1b1539STejun Heo * @rq: request to insert 404ae1b1539STejun Heo * 405b710a480SJens Axboe * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions. 406320ae51fSJens Axboe * or __blk_mq_run_hw_queue() to dispatch request. 407ae1b1539STejun Heo * @rq is being submitted. Analyze what needs to be done and put it on the 408ae1b1539STejun Heo * right queue. 409ae1b1539STejun Heo * 410ae1b1539STejun Heo * CONTEXT: 411320ae51fSJens Axboe * spin_lock_irq(q->queue_lock) in !mq case 412ae1b1539STejun Heo */ 413ae1b1539STejun Heo void blk_insert_flush(struct request *rq) 414ae1b1539STejun Heo { 415ae1b1539STejun Heo struct request_queue *q = rq->q; 416c888a8f9SJens Axboe unsigned long fflags = q->queue_flags; /* may change, cache */ 417ae1b1539STejun Heo unsigned int policy = blk_flush_policy(fflags, rq); 418e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx); 419ae1b1539STejun Heo 420ae1b1539STejun Heo /* 421ae1b1539STejun Heo * @policy now records what operations need to be done. Adjust 42228a8f0d3SMike Christie * REQ_PREFLUSH and FUA for the driver. 423ae1b1539STejun Heo */ 42428a8f0d3SMike Christie rq->cmd_flags &= ~REQ_PREFLUSH; 425c888a8f9SJens Axboe if (!(fflags & (1UL << QUEUE_FLAG_FUA))) 4264fed947cSTejun Heo rq->cmd_flags &= ~REQ_FUA; 427ae1b1539STejun Heo 428ae1b1539STejun Heo /* 429ae5b2ec8SJens Axboe * REQ_PREFLUSH|REQ_FUA implies REQ_SYNC, so if we clear any 430ae5b2ec8SJens Axboe * of those flags, we have to set REQ_SYNC to avoid skewing 431ae5b2ec8SJens Axboe * the request accounting. 432ae5b2ec8SJens Axboe */ 433ae5b2ec8SJens Axboe rq->cmd_flags |= REQ_SYNC; 434ae5b2ec8SJens Axboe 435ae5b2ec8SJens Axboe /* 4364853abaaSJeff Moyer * An empty flush handed down from a stacking driver may 4374853abaaSJeff Moyer * translate into nothing if the underlying device does not 4384853abaaSJeff Moyer * advertise a write-back cache. In this case, simply 4394853abaaSJeff Moyer * complete the request. 4404853abaaSJeff Moyer */ 4414853abaaSJeff Moyer if (!policy) { 442320ae51fSJens Axboe if (q->mq_ops) 443c8a446adSChristoph Hellwig blk_mq_end_request(rq, 0); 444320ae51fSJens Axboe else 4454853abaaSJeff Moyer __blk_end_bidi_request(rq, 0, 0, 0); 4464853abaaSJeff Moyer return; 4474853abaaSJeff Moyer } 4484853abaaSJeff Moyer 449834f9f61SJeff Moyer BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */ 4504853abaaSJeff Moyer 4514853abaaSJeff Moyer /* 452ae1b1539STejun Heo * If there's data but flush is not necessary, the request can be 453ae1b1539STejun Heo * processed directly without going through flush machinery. Queue 454ae1b1539STejun Heo * for normal execution. 455ae1b1539STejun Heo */ 456ae1b1539STejun Heo if ((policy & REQ_FSEQ_DATA) && 457ae1b1539STejun Heo !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) { 458bd166ef1SJens Axboe if (q->mq_ops) 459bd6737f1SJens Axboe blk_mq_sched_insert_request(rq, false, true, false, false); 460bd166ef1SJens Axboe else 461dcd8376cSJens Axboe list_add_tail(&rq->queuelist, &q->queue_head); 462ae1b1539STejun Heo return; 4638839a0e0STejun Heo } 4648839a0e0STejun Heo 4658839a0e0STejun Heo /* 466ae1b1539STejun Heo * @rq should go through flush machinery. Mark it part of flush 467ae1b1539STejun Heo * sequence and submit for further processing. 4688839a0e0STejun Heo */ 469ae1b1539STejun Heo memset(&rq->flush, 0, sizeof(rq->flush)); 470ae1b1539STejun Heo INIT_LIST_HEAD(&rq->flush.list); 471e8064021SChristoph Hellwig rq->rq_flags |= RQF_FLUSH_SEQ; 4724853abaaSJeff Moyer rq->flush.saved_end_io = rq->end_io; /* Usually NULL */ 473320ae51fSJens Axboe if (q->mq_ops) { 474320ae51fSJens Axboe rq->end_io = mq_flush_data_end_io; 475320ae51fSJens Axboe 4767c94e1c1SMing Lei spin_lock_irq(&fq->mq_flush_lock); 4770bae352dSMing Lei blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); 4787c94e1c1SMing Lei spin_unlock_irq(&fq->mq_flush_lock); 479320ae51fSJens Axboe return; 480320ae51fSJens Axboe } 481ae1b1539STejun Heo rq->end_io = flush_data_end_io; 482ae1b1539STejun Heo 4830bae352dSMing Lei blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); 484ae1b1539STejun Heo } 485ae1b1539STejun Heo 486ae1b1539STejun Heo /** 4878839a0e0STejun Heo * blkdev_issue_flush - queue a flush 4888839a0e0STejun Heo * @bdev: blockdev to issue flush for 4898839a0e0STejun Heo * @gfp_mask: memory allocation flags (for bio_alloc) 4908839a0e0STejun Heo * @error_sector: error sector 4918839a0e0STejun Heo * 4928839a0e0STejun Heo * Description: 4938839a0e0STejun Heo * Issue a flush for the block device in question. Caller can supply 4948839a0e0STejun Heo * room for storing the error offset in case of a flush error, if they 4958839a0e0STejun Heo * wish to. If WAIT flag is not passed then caller may check only what 4968839a0e0STejun Heo * request was pushed in some internal queue for later handling. 4978839a0e0STejun Heo */ 4988839a0e0STejun Heo int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, 499dd3932edSChristoph Hellwig sector_t *error_sector) 5008839a0e0STejun Heo { 5018839a0e0STejun Heo struct request_queue *q; 5028839a0e0STejun Heo struct bio *bio; 5038839a0e0STejun Heo int ret = 0; 5048839a0e0STejun Heo 5058839a0e0STejun Heo if (bdev->bd_disk == NULL) 5068839a0e0STejun Heo return -ENXIO; 5078839a0e0STejun Heo 5088839a0e0STejun Heo q = bdev_get_queue(bdev); 5098839a0e0STejun Heo if (!q) 5108839a0e0STejun Heo return -ENXIO; 5118839a0e0STejun Heo 5128839a0e0STejun Heo /* 5138839a0e0STejun Heo * some block devices may not have their queue correctly set up here 5148839a0e0STejun Heo * (e.g. loop device without a backing file) and so issuing a flush 5158839a0e0STejun Heo * here will panic. Ensure there is a request function before issuing 516d391a2ddSTejun Heo * the flush. 5178839a0e0STejun Heo */ 5188839a0e0STejun Heo if (!q->make_request_fn) 5198839a0e0STejun Heo return -ENXIO; 5208839a0e0STejun Heo 5218839a0e0STejun Heo bio = bio_alloc(gfp_mask, 0); 5228839a0e0STejun Heo bio->bi_bdev = bdev; 52370fd7614SChristoph Hellwig bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; 5248839a0e0STejun Heo 5254e49ea4aSMike Christie ret = submit_bio_wait(bio); 526dd3932edSChristoph Hellwig 5278839a0e0STejun Heo /* 5288839a0e0STejun Heo * The driver must store the error location in ->bi_sector, if 5298839a0e0STejun Heo * it supports it. For non-stacked drivers, this should be 5308839a0e0STejun Heo * copied from blk_rq_pos(rq). 5318839a0e0STejun Heo */ 5328839a0e0STejun Heo if (error_sector) 5334f024f37SKent Overstreet *error_sector = bio->bi_iter.bi_sector; 5348839a0e0STejun Heo 5358839a0e0STejun Heo bio_put(bio); 5368839a0e0STejun Heo return ret; 5378839a0e0STejun Heo } 5388839a0e0STejun Heo EXPORT_SYMBOL(blkdev_issue_flush); 539320ae51fSJens Axboe 540f70ced09SMing Lei struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, 541f70ced09SMing Lei int node, int cmd_size) 542320ae51fSJens Axboe { 5437c94e1c1SMing Lei struct blk_flush_queue *fq; 5447c94e1c1SMing Lei int rq_sz = sizeof(struct request); 5451bcb1eadSMing Lei 546f70ced09SMing Lei fq = kzalloc_node(sizeof(*fq), GFP_KERNEL, node); 5477c94e1c1SMing Lei if (!fq) 5487c94e1c1SMing Lei goto fail; 5491bcb1eadSMing Lei 5507c94e1c1SMing Lei if (q->mq_ops) { 5517c94e1c1SMing Lei spin_lock_init(&fq->mq_flush_lock); 552f70ced09SMing Lei rq_sz = round_up(rq_sz + cmd_size, cache_line_size()); 5537c94e1c1SMing Lei } 5547c94e1c1SMing Lei 555f70ced09SMing Lei fq->flush_rq = kzalloc_node(rq_sz, GFP_KERNEL, node); 5567c94e1c1SMing Lei if (!fq->flush_rq) 5577c94e1c1SMing Lei goto fail_rq; 5587c94e1c1SMing Lei 5597c94e1c1SMing Lei INIT_LIST_HEAD(&fq->flush_queue[0]); 5607c94e1c1SMing Lei INIT_LIST_HEAD(&fq->flush_queue[1]); 5617c94e1c1SMing Lei INIT_LIST_HEAD(&fq->flush_data_in_flight); 5627c94e1c1SMing Lei 5637c94e1c1SMing Lei return fq; 5647c94e1c1SMing Lei 5657c94e1c1SMing Lei fail_rq: 5667c94e1c1SMing Lei kfree(fq); 5677c94e1c1SMing Lei fail: 5687c94e1c1SMing Lei return NULL; 5697c94e1c1SMing Lei } 5707c94e1c1SMing Lei 571ba483388SMing Lei void blk_free_flush_queue(struct blk_flush_queue *fq) 5727c94e1c1SMing Lei { 5737c94e1c1SMing Lei /* bio based request queue hasn't flush queue */ 5747c94e1c1SMing Lei if (!fq) 5757c94e1c1SMing Lei return; 5767c94e1c1SMing Lei 5777c94e1c1SMing Lei kfree(fq->flush_rq); 5787c94e1c1SMing Lei kfree(fq); 579320ae51fSJens Axboe } 580