18839a0e0STejun Heo /* 24fed947cSTejun Heo * Functions to sequence FLUSH and FUA writes. 3ae1b1539STejun Heo * 4ae1b1539STejun Heo * Copyright (C) 2011 Max Planck Institute for Gravitational Physics 5ae1b1539STejun Heo * Copyright (C) 2011 Tejun Heo <tj@kernel.org> 6ae1b1539STejun Heo * 7ae1b1539STejun Heo * This file is released under the GPLv2. 8ae1b1539STejun Heo * 9ae1b1539STejun Heo * REQ_{FLUSH|FUA} requests are decomposed to sequences consisted of three 10ae1b1539STejun Heo * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request 11ae1b1539STejun Heo * properties and hardware capability. 12ae1b1539STejun Heo * 1328a8f0d3SMike Christie * If a request doesn't have data, only REQ_PREFLUSH makes sense, which 1428a8f0d3SMike Christie * indicates a simple flush request. If there is data, REQ_PREFLUSH indicates 15ae1b1539STejun Heo * that the device cache should be flushed before the data is executed, and 16ae1b1539STejun Heo * REQ_FUA means that the data must be on non-volatile media on request 17ae1b1539STejun Heo * completion. 18ae1b1539STejun Heo * 19ae1b1539STejun Heo * If the device doesn't have writeback cache, FLUSH and FUA don't make any 20ae1b1539STejun Heo * difference. The requests are either completed immediately if there's no 21ae1b1539STejun Heo * data or executed as normal requests otherwise. 22ae1b1539STejun Heo * 2328a8f0d3SMike Christie * If the device has writeback cache and supports FUA, REQ_PREFLUSH is 24ae1b1539STejun Heo * translated to PREFLUSH but REQ_FUA is passed down directly with DATA. 25ae1b1539STejun Heo * 2628a8f0d3SMike Christie * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH 2728a8f0d3SMike Christie * is translated to PREFLUSH and REQ_FUA to POSTFLUSH. 28ae1b1539STejun Heo * 29ae1b1539STejun Heo * The actual execution of flush is double buffered. Whenever a request 30ae1b1539STejun Heo * needs to execute PRE or POSTFLUSH, it queues at 317c94e1c1SMing Lei * fq->flush_queue[fq->flush_pending_idx]. Once certain criteria are met, a 323a5e02ceSMike Christie * REQ_OP_FLUSH is issued and the pending_idx is toggled. When the flush 33ae1b1539STejun Heo * completes, all the requests which were pending are proceeded to the next 34ae1b1539STejun Heo * step. This allows arbitrary merging of different types of FLUSH/FUA 35ae1b1539STejun Heo * requests. 36ae1b1539STejun Heo * 37ae1b1539STejun Heo * Currently, the following conditions are used to determine when to issue 38ae1b1539STejun Heo * flush. 39ae1b1539STejun Heo * 40ae1b1539STejun Heo * C1. At any given time, only one flush shall be in progress. This makes 41ae1b1539STejun Heo * double buffering sufficient. 42ae1b1539STejun Heo * 43ae1b1539STejun Heo * C2. Flush is deferred if any request is executing DATA of its sequence. 44ae1b1539STejun Heo * This avoids issuing separate POSTFLUSHes for requests which shared 45ae1b1539STejun Heo * PREFLUSH. 46ae1b1539STejun Heo * 47ae1b1539STejun Heo * C3. The second condition is ignored if there is a request which has 48ae1b1539STejun Heo * waited longer than FLUSH_PENDING_TIMEOUT. This is to avoid 49ae1b1539STejun Heo * starvation in the unlikely case where there are continuous stream of 50ae1b1539STejun Heo * FUA (without FLUSH) requests. 51ae1b1539STejun Heo * 52ae1b1539STejun Heo * For devices which support FUA, it isn't clear whether C2 (and thus C3) 53ae1b1539STejun Heo * is beneficial. 54ae1b1539STejun Heo * 55ae1b1539STejun Heo * Note that a sequenced FLUSH/FUA request with DATA is completed twice. 56ae1b1539STejun Heo * Once while executing DATA and again after the whole sequence is 57ae1b1539STejun Heo * complete. The first completion updates the contained bio but doesn't 58ae1b1539STejun Heo * finish it so that the bio submitter is notified only after the whole 59e8064021SChristoph Hellwig * sequence is complete. This is implemented by testing RQF_FLUSH_SEQ in 60ae1b1539STejun Heo * req_bio_endio(). 61ae1b1539STejun Heo * 62ae1b1539STejun Heo * The above peculiarity requires that each FLUSH/FUA request has only one 63ae1b1539STejun Heo * bio attached to it, which is guaranteed as they aren't allowed to be 64ae1b1539STejun Heo * merged in the usual way. 658839a0e0STejun Heo */ 66ae1b1539STejun Heo 678839a0e0STejun Heo #include <linux/kernel.h> 688839a0e0STejun Heo #include <linux/module.h> 698839a0e0STejun Heo #include <linux/bio.h> 708839a0e0STejun Heo #include <linux/blkdev.h> 718839a0e0STejun Heo #include <linux/gfp.h> 72320ae51fSJens Axboe #include <linux/blk-mq.h> 738839a0e0STejun Heo 748839a0e0STejun Heo #include "blk.h" 75320ae51fSJens Axboe #include "blk-mq.h" 760048b483SMing Lei #include "blk-mq-tag.h" 77bd166ef1SJens Axboe #include "blk-mq-sched.h" 788839a0e0STejun Heo 794fed947cSTejun Heo /* FLUSH/FUA sequences */ 804fed947cSTejun Heo enum { 81ae1b1539STejun Heo REQ_FSEQ_PREFLUSH = (1 << 0), /* pre-flushing in progress */ 82ae1b1539STejun Heo REQ_FSEQ_DATA = (1 << 1), /* data write in progress */ 83ae1b1539STejun Heo REQ_FSEQ_POSTFLUSH = (1 << 2), /* post-flushing in progress */ 84ae1b1539STejun Heo REQ_FSEQ_DONE = (1 << 3), 85ae1b1539STejun Heo 86ae1b1539STejun Heo REQ_FSEQ_ACTIONS = REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA | 87ae1b1539STejun Heo REQ_FSEQ_POSTFLUSH, 88ae1b1539STejun Heo 89ae1b1539STejun Heo /* 90ae1b1539STejun Heo * If flush has been pending longer than the following timeout, 91ae1b1539STejun Heo * it's issued even if flush_data requests are still in flight. 92ae1b1539STejun Heo */ 93ae1b1539STejun Heo FLUSH_PENDING_TIMEOUT = 5 * HZ, 944fed947cSTejun Heo }; 954fed947cSTejun Heo 960bae352dSMing Lei static bool blk_kick_flush(struct request_queue *q, 970bae352dSMing Lei struct blk_flush_queue *fq); 988839a0e0STejun Heo 99c888a8f9SJens Axboe static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq) 1008839a0e0STejun Heo { 101ae1b1539STejun Heo unsigned int policy = 0; 102ae1b1539STejun Heo 103fa1bf42fSJeff Moyer if (blk_rq_sectors(rq)) 104fa1bf42fSJeff Moyer policy |= REQ_FSEQ_DATA; 105fa1bf42fSJeff Moyer 106c888a8f9SJens Axboe if (fflags & (1UL << QUEUE_FLAG_WC)) { 10728a8f0d3SMike Christie if (rq->cmd_flags & REQ_PREFLUSH) 108ae1b1539STejun Heo policy |= REQ_FSEQ_PREFLUSH; 109c888a8f9SJens Axboe if (!(fflags & (1UL << QUEUE_FLAG_FUA)) && 110c888a8f9SJens Axboe (rq->cmd_flags & REQ_FUA)) 111ae1b1539STejun Heo policy |= REQ_FSEQ_POSTFLUSH; 112ae1b1539STejun Heo } 113ae1b1539STejun Heo return policy; 1148839a0e0STejun Heo } 1158839a0e0STejun Heo 116ae1b1539STejun Heo static unsigned int blk_flush_cur_seq(struct request *rq) 1178839a0e0STejun Heo { 118ae1b1539STejun Heo return 1 << ffz(rq->flush.seq); 1198839a0e0STejun Heo } 1208839a0e0STejun Heo 121ae1b1539STejun Heo static void blk_flush_restore_request(struct request *rq) 12247f70d5aSTejun Heo { 12347f70d5aSTejun Heo /* 124ae1b1539STejun Heo * After flush data completion, @rq->bio is %NULL but we need to 125ae1b1539STejun Heo * complete the bio again. @rq->biotail is guaranteed to equal the 126ae1b1539STejun Heo * original @rq->bio. Restore it. 12747f70d5aSTejun Heo */ 128ae1b1539STejun Heo rq->bio = rq->biotail; 129ae1b1539STejun Heo 130ae1b1539STejun Heo /* make @rq a normal request */ 131e8064021SChristoph Hellwig rq->rq_flags &= ~RQF_FLUSH_SEQ; 1324853abaaSJeff Moyer rq->end_io = rq->flush.saved_end_io; 133320ae51fSJens Axboe } 134320ae51fSJens Axboe 13510beafc1SMike Snitzer static bool blk_flush_queue_rq(struct request *rq, bool add_front) 136320ae51fSJens Axboe { 13718741986SChristoph Hellwig if (rq->q->mq_ops) { 1382b053acaSBart Van Assche blk_mq_add_to_requeue_list(rq, add_front, true); 13918741986SChristoph Hellwig return false; 14018741986SChristoph Hellwig } else { 14110beafc1SMike Snitzer if (add_front) 14210beafc1SMike Snitzer list_add(&rq->queuelist, &rq->q->queue_head); 14310beafc1SMike Snitzer else 14418741986SChristoph Hellwig list_add_tail(&rq->queuelist, &rq->q->queue_head); 14518741986SChristoph Hellwig return true; 14618741986SChristoph Hellwig } 14747f70d5aSTejun Heo } 14847f70d5aSTejun Heo 149ae1b1539STejun Heo /** 150ae1b1539STejun Heo * blk_flush_complete_seq - complete flush sequence 151ae1b1539STejun Heo * @rq: FLUSH/FUA request being sequenced 1520bae352dSMing Lei * @fq: flush queue 153ae1b1539STejun Heo * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero) 154ae1b1539STejun Heo * @error: whether an error occurred 155ae1b1539STejun Heo * 156ae1b1539STejun Heo * @rq just completed @seq part of its flush sequence, record the 157ae1b1539STejun Heo * completion and trigger the next step. 158ae1b1539STejun Heo * 159ae1b1539STejun Heo * CONTEXT: 1607c94e1c1SMing Lei * spin_lock_irq(q->queue_lock or fq->mq_flush_lock) 161ae1b1539STejun Heo * 162ae1b1539STejun Heo * RETURNS: 163ae1b1539STejun Heo * %true if requests were added to the dispatch queue, %false otherwise. 164ae1b1539STejun Heo */ 1650bae352dSMing Lei static bool blk_flush_complete_seq(struct request *rq, 1660bae352dSMing Lei struct blk_flush_queue *fq, 1670bae352dSMing Lei unsigned int seq, int error) 1688839a0e0STejun Heo { 169ae1b1539STejun Heo struct request_queue *q = rq->q; 1707c94e1c1SMing Lei struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; 171320ae51fSJens Axboe bool queued = false, kicked; 1728839a0e0STejun Heo 173ae1b1539STejun Heo BUG_ON(rq->flush.seq & seq); 174ae1b1539STejun Heo rq->flush.seq |= seq; 1758839a0e0STejun Heo 176ae1b1539STejun Heo if (likely(!error)) 177ae1b1539STejun Heo seq = blk_flush_cur_seq(rq); 178ae1b1539STejun Heo else 179ae1b1539STejun Heo seq = REQ_FSEQ_DONE; 1808839a0e0STejun Heo 181ae1b1539STejun Heo switch (seq) { 182ae1b1539STejun Heo case REQ_FSEQ_PREFLUSH: 183ae1b1539STejun Heo case REQ_FSEQ_POSTFLUSH: 184ae1b1539STejun Heo /* queue for flush */ 185ae1b1539STejun Heo if (list_empty(pending)) 1867c94e1c1SMing Lei fq->flush_pending_since = jiffies; 187ae1b1539STejun Heo list_move_tail(&rq->flush.list, pending); 1888839a0e0STejun Heo break; 189ae1b1539STejun Heo 190ae1b1539STejun Heo case REQ_FSEQ_DATA: 1917c94e1c1SMing Lei list_move_tail(&rq->flush.list, &fq->flush_data_in_flight); 19210beafc1SMike Snitzer queued = blk_flush_queue_rq(rq, true); 193ae1b1539STejun Heo break; 194ae1b1539STejun Heo 195ae1b1539STejun Heo case REQ_FSEQ_DONE: 19609d60c70STejun Heo /* 197ae1b1539STejun Heo * @rq was previously adjusted by blk_flush_issue() for 198ae1b1539STejun Heo * flush sequencing and may already have gone through the 199ae1b1539STejun Heo * flush data request completion path. Restore @rq for 200ae1b1539STejun Heo * normal completion and end it. 20109d60c70STejun Heo */ 202ae1b1539STejun Heo BUG_ON(!list_empty(&rq->queuelist)); 203ae1b1539STejun Heo list_del_init(&rq->flush.list); 204ae1b1539STejun Heo blk_flush_restore_request(rq); 205320ae51fSJens Axboe if (q->mq_ops) 206c8a446adSChristoph Hellwig blk_mq_end_request(rq, error); 207320ae51fSJens Axboe else 208ae1b1539STejun Heo __blk_end_request_all(rq, error); 2098839a0e0STejun Heo break; 210ae1b1539STejun Heo 2118839a0e0STejun Heo default: 2128839a0e0STejun Heo BUG(); 2138839a0e0STejun Heo } 214cde4c406SChristoph Hellwig 2150bae352dSMing Lei kicked = blk_kick_flush(q, fq); 216320ae51fSJens Axboe return kicked | queued; 2178839a0e0STejun Heo } 2188839a0e0STejun Heo 219ae1b1539STejun Heo static void flush_end_io(struct request *flush_rq, int error) 2208839a0e0STejun Heo { 221ae1b1539STejun Heo struct request_queue *q = flush_rq->q; 222320ae51fSJens Axboe struct list_head *running; 223ae1b1539STejun Heo bool queued = false; 224ae1b1539STejun Heo struct request *rq, *n; 225320ae51fSJens Axboe unsigned long flags = 0; 226e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx); 2278839a0e0STejun Heo 22822302375SShaohua Li if (q->mq_ops) { 2290048b483SMing Lei struct blk_mq_hw_ctx *hctx; 2300048b483SMing Lei 2310048b483SMing Lei /* release the tag's ownership to the req cloned from */ 2327c94e1c1SMing Lei spin_lock_irqsave(&fq->mq_flush_lock, flags); 2337d7e0f90SChristoph Hellwig hctx = blk_mq_map_queue(q, flush_rq->mq_ctx->cpu); 2340048b483SMing Lei blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq); 2357ddab5deSMing Lei flush_rq->tag = -1; 23622302375SShaohua Li } 23718741986SChristoph Hellwig 2387c94e1c1SMing Lei running = &fq->flush_queue[fq->flush_running_idx]; 2397c94e1c1SMing Lei BUG_ON(fq->flush_pending_idx == fq->flush_running_idx); 240ae1b1539STejun Heo 241ae1b1539STejun Heo /* account completion of the flush request */ 2427c94e1c1SMing Lei fq->flush_running_idx ^= 1; 243320ae51fSJens Axboe 244320ae51fSJens Axboe if (!q->mq_ops) 245ae1b1539STejun Heo elv_completed_request(q, flush_rq); 246ae1b1539STejun Heo 247ae1b1539STejun Heo /* and push the waiting requests to the next stage */ 248ae1b1539STejun Heo list_for_each_entry_safe(rq, n, running, flush.list) { 249ae1b1539STejun Heo unsigned int seq = blk_flush_cur_seq(rq); 250ae1b1539STejun Heo 251ae1b1539STejun Heo BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH); 2520bae352dSMing Lei queued |= blk_flush_complete_seq(rq, fq, seq, error); 253ae1b1539STejun Heo } 254ae1b1539STejun Heo 2558839a0e0STejun Heo /* 2563ac0cc45Sshaohua.li@intel.com * Kick the queue to avoid stall for two cases: 2573ac0cc45Sshaohua.li@intel.com * 1. Moving a request silently to empty queue_head may stall the 2583ac0cc45Sshaohua.li@intel.com * queue. 2593ac0cc45Sshaohua.li@intel.com * 2. When flush request is running in non-queueable queue, the 2603ac0cc45Sshaohua.li@intel.com * queue is hold. Restart the queue after flush request is finished 2613ac0cc45Sshaohua.li@intel.com * to avoid stall. 2623ac0cc45Sshaohua.li@intel.com * This function is called from request completion path and calling 2633ac0cc45Sshaohua.li@intel.com * directly into request_fn may confuse the driver. Always use 2643ac0cc45Sshaohua.li@intel.com * kblockd. 2658839a0e0STejun Heo */ 2667c94e1c1SMing Lei if (queued || fq->flush_queue_delayed) { 26718741986SChristoph Hellwig WARN_ON(q->mq_ops); 26824ecfbe2SChristoph Hellwig blk_run_queue_async(q); 269320ae51fSJens Axboe } 2707c94e1c1SMing Lei fq->flush_queue_delayed = 0; 271320ae51fSJens Axboe if (q->mq_ops) 2727c94e1c1SMing Lei spin_unlock_irqrestore(&fq->mq_flush_lock, flags); 273320ae51fSJens Axboe } 274320ae51fSJens Axboe 275ae1b1539STejun Heo /** 276ae1b1539STejun Heo * blk_kick_flush - consider issuing flush request 277ae1b1539STejun Heo * @q: request_queue being kicked 2780bae352dSMing Lei * @fq: flush queue 2794fed947cSTejun Heo * 280ae1b1539STejun Heo * Flush related states of @q have changed, consider issuing flush request. 281ae1b1539STejun Heo * Please read the comment at the top of this file for more info. 282ae1b1539STejun Heo * 283ae1b1539STejun Heo * CONTEXT: 2847c94e1c1SMing Lei * spin_lock_irq(q->queue_lock or fq->mq_flush_lock) 285ae1b1539STejun Heo * 286ae1b1539STejun Heo * RETURNS: 287ae1b1539STejun Heo * %true if flush was issued, %false otherwise. 2888839a0e0STejun Heo */ 2890bae352dSMing Lei static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) 290ae1b1539STejun Heo { 2917c94e1c1SMing Lei struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx]; 292ae1b1539STejun Heo struct request *first_rq = 293ae1b1539STejun Heo list_first_entry(pending, struct request, flush.list); 2947c94e1c1SMing Lei struct request *flush_rq = fq->flush_rq; 295ae1b1539STejun Heo 296ae1b1539STejun Heo /* C1 described at the top of this file */ 2977c94e1c1SMing Lei if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending)) 298ae1b1539STejun Heo return false; 299ae1b1539STejun Heo 3007520872cSJens Axboe /* C2 and C3 3017520872cSJens Axboe * 3027520872cSJens Axboe * For blk-mq + scheduling, we can risk having all driver tags 3037520872cSJens Axboe * assigned to empty flushes, and we deadlock if we are expecting 3047520872cSJens Axboe * other requests to make progress. Don't defer for that case. 3057520872cSJens Axboe */ 3067c94e1c1SMing Lei if (!list_empty(&fq->flush_data_in_flight) && 3077520872cSJens Axboe !(q->mq_ops && q->elevator) && 308ae1b1539STejun Heo time_before(jiffies, 3097c94e1c1SMing Lei fq->flush_pending_since + FLUSH_PENDING_TIMEOUT)) 310ae1b1539STejun Heo return false; 311ae1b1539STejun Heo 312ae1b1539STejun Heo /* 313ae1b1539STejun Heo * Issue flush and toggle pending_idx. This makes pending_idx 314ae1b1539STejun Heo * different from running_idx, which means flush is in flight. 315ae1b1539STejun Heo */ 3167c94e1c1SMing Lei fq->flush_pending_idx ^= 1; 31718741986SChristoph Hellwig 3187ddab5deSMing Lei blk_rq_init(q, flush_rq); 319f70ced09SMing Lei 320f70ced09SMing Lei /* 321f70ced09SMing Lei * Borrow tag from the first request since they can't 3220048b483SMing Lei * be in flight at the same time. And acquire the tag's 3230048b483SMing Lei * ownership for flush req. 324f70ced09SMing Lei */ 325f70ced09SMing Lei if (q->mq_ops) { 3260048b483SMing Lei struct blk_mq_hw_ctx *hctx; 3270048b483SMing Lei 328f70ced09SMing Lei flush_rq->mq_ctx = first_rq->mq_ctx; 329f70ced09SMing Lei flush_rq->tag = first_rq->tag; 3300048b483SMing Lei fq->orig_rq = first_rq; 3310048b483SMing Lei 3327d7e0f90SChristoph Hellwig hctx = blk_mq_map_queue(q, first_rq->mq_ctx->cpu); 3330048b483SMing Lei blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq); 334f70ced09SMing Lei } 335320ae51fSJens Axboe 33670fd7614SChristoph Hellwig flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH; 337e8064021SChristoph Hellwig flush_rq->rq_flags |= RQF_FLUSH_SEQ; 3387ddab5deSMing Lei flush_rq->rq_disk = first_rq->rq_disk; 3397ddab5deSMing Lei flush_rq->end_io = flush_end_io; 340ae1b1539STejun Heo 3417ddab5deSMing Lei return blk_flush_queue_rq(flush_rq, false); 342ae1b1539STejun Heo } 343ae1b1539STejun Heo 344ae1b1539STejun Heo static void flush_data_end_io(struct request *rq, int error) 345ae1b1539STejun Heo { 346ae1b1539STejun Heo struct request_queue *q = rq->q; 347e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); 348ae1b1539STejun Heo 3498839a0e0STejun Heo /* 35094d7dea4SMing Lei * Updating q->in_flight[] here for making this tag usable 35194d7dea4SMing Lei * early. Because in blk_queue_start_tag(), 35294d7dea4SMing Lei * q->in_flight[BLK_RW_ASYNC] is used to limit async I/O and 35394d7dea4SMing Lei * reserve tags for sync I/O. 35494d7dea4SMing Lei * 35594d7dea4SMing Lei * More importantly this way can avoid the following I/O 35694d7dea4SMing Lei * deadlock: 35794d7dea4SMing Lei * 35894d7dea4SMing Lei * - suppose there are 40 fua requests comming to flush queue 35994d7dea4SMing Lei * and queue depth is 31 36094d7dea4SMing Lei * - 30 rqs are scheduled then blk_queue_start_tag() can't alloc 36194d7dea4SMing Lei * tag for async I/O any more 36294d7dea4SMing Lei * - all the 30 rqs are completed before FLUSH_PENDING_TIMEOUT 36394d7dea4SMing Lei * and flush_data_end_io() is called 36494d7dea4SMing Lei * - the other rqs still can't go ahead if not updating 36594d7dea4SMing Lei * q->in_flight[BLK_RW_ASYNC] here, meantime these rqs 36694d7dea4SMing Lei * are held in flush data queue and make no progress of 36794d7dea4SMing Lei * handling post flush rq 36894d7dea4SMing Lei * - only after the post flush rq is handled, all these rqs 36994d7dea4SMing Lei * can be completed 37094d7dea4SMing Lei */ 37194d7dea4SMing Lei 37294d7dea4SMing Lei elv_completed_request(q, rq); 37394d7dea4SMing Lei 37494d7dea4SMing Lei /* for avoiding double accounting */ 37536869cb9SLinus Torvalds rq->rq_flags &= ~RQF_STARTED; 37694d7dea4SMing Lei 37794d7dea4SMing Lei /* 378e83a46bbSTejun Heo * After populating an empty queue, kick it to avoid stall. Read 379e83a46bbSTejun Heo * the comment in flush_end_io(). 3808839a0e0STejun Heo */ 3810bae352dSMing Lei if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error)) 38224ecfbe2SChristoph Hellwig blk_run_queue_async(q); 383ae1b1539STejun Heo } 384ae1b1539STejun Heo 385320ae51fSJens Axboe static void mq_flush_data_end_io(struct request *rq, int error) 386320ae51fSJens Axboe { 387320ae51fSJens Axboe struct request_queue *q = rq->q; 388320ae51fSJens Axboe struct blk_mq_hw_ctx *hctx; 389e97c293cSMing Lei struct blk_mq_ctx *ctx = rq->mq_ctx; 390320ae51fSJens Axboe unsigned long flags; 391e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx); 392320ae51fSJens Axboe 3937d7e0f90SChristoph Hellwig hctx = blk_mq_map_queue(q, ctx->cpu); 394320ae51fSJens Axboe 395320ae51fSJens Axboe /* 396320ae51fSJens Axboe * After populating an empty queue, kick it to avoid stall. Read 397320ae51fSJens Axboe * the comment in flush_end_io(). 398320ae51fSJens Axboe */ 3997c94e1c1SMing Lei spin_lock_irqsave(&fq->mq_flush_lock, flags); 400bd166ef1SJens Axboe blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error); 4017c94e1c1SMing Lei spin_unlock_irqrestore(&fq->mq_flush_lock, flags); 402bd166ef1SJens Axboe 403bd166ef1SJens Axboe blk_mq_run_hw_queue(hctx, true); 404320ae51fSJens Axboe } 405320ae51fSJens Axboe 406ae1b1539STejun Heo /** 407ae1b1539STejun Heo * blk_insert_flush - insert a new FLUSH/FUA request 408ae1b1539STejun Heo * @rq: request to insert 409ae1b1539STejun Heo * 410b710a480SJens Axboe * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions. 411320ae51fSJens Axboe * or __blk_mq_run_hw_queue() to dispatch request. 412ae1b1539STejun Heo * @rq is being submitted. Analyze what needs to be done and put it on the 413ae1b1539STejun Heo * right queue. 414ae1b1539STejun Heo * 415ae1b1539STejun Heo * CONTEXT: 416320ae51fSJens Axboe * spin_lock_irq(q->queue_lock) in !mq case 417ae1b1539STejun Heo */ 418ae1b1539STejun Heo void blk_insert_flush(struct request *rq) 419ae1b1539STejun Heo { 420ae1b1539STejun Heo struct request_queue *q = rq->q; 421c888a8f9SJens Axboe unsigned long fflags = q->queue_flags; /* may change, cache */ 422ae1b1539STejun Heo unsigned int policy = blk_flush_policy(fflags, rq); 423e97c293cSMing Lei struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx); 424ae1b1539STejun Heo 425ae1b1539STejun Heo /* 426ae1b1539STejun Heo * @policy now records what operations need to be done. Adjust 42728a8f0d3SMike Christie * REQ_PREFLUSH and FUA for the driver. 428ae1b1539STejun Heo */ 42928a8f0d3SMike Christie rq->cmd_flags &= ~REQ_PREFLUSH; 430c888a8f9SJens Axboe if (!(fflags & (1UL << QUEUE_FLAG_FUA))) 4314fed947cSTejun Heo rq->cmd_flags &= ~REQ_FUA; 432ae1b1539STejun Heo 433ae1b1539STejun Heo /* 434ae5b2ec8SJens Axboe * REQ_PREFLUSH|REQ_FUA implies REQ_SYNC, so if we clear any 435ae5b2ec8SJens Axboe * of those flags, we have to set REQ_SYNC to avoid skewing 436ae5b2ec8SJens Axboe * the request accounting. 437ae5b2ec8SJens Axboe */ 438ae5b2ec8SJens Axboe rq->cmd_flags |= REQ_SYNC; 439ae5b2ec8SJens Axboe 440ae5b2ec8SJens Axboe /* 4414853abaaSJeff Moyer * An empty flush handed down from a stacking driver may 4424853abaaSJeff Moyer * translate into nothing if the underlying device does not 4434853abaaSJeff Moyer * advertise a write-back cache. In this case, simply 4444853abaaSJeff Moyer * complete the request. 4454853abaaSJeff Moyer */ 4464853abaaSJeff Moyer if (!policy) { 447320ae51fSJens Axboe if (q->mq_ops) 448c8a446adSChristoph Hellwig blk_mq_end_request(rq, 0); 449320ae51fSJens Axboe else 450d0fac025SChristoph Hellwig __blk_end_request(rq, 0, 0); 4514853abaaSJeff Moyer return; 4524853abaaSJeff Moyer } 4534853abaaSJeff Moyer 454834f9f61SJeff Moyer BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */ 4554853abaaSJeff Moyer 4564853abaaSJeff Moyer /* 457ae1b1539STejun Heo * If there's data but flush is not necessary, the request can be 458ae1b1539STejun Heo * processed directly without going through flush machinery. Queue 459ae1b1539STejun Heo * for normal execution. 460ae1b1539STejun Heo */ 461ae1b1539STejun Heo if ((policy & REQ_FSEQ_DATA) && 462ae1b1539STejun Heo !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) { 463bd166ef1SJens Axboe if (q->mq_ops) 464bd6737f1SJens Axboe blk_mq_sched_insert_request(rq, false, true, false, false); 465bd166ef1SJens Axboe else 466dcd8376cSJens Axboe list_add_tail(&rq->queuelist, &q->queue_head); 467ae1b1539STejun Heo return; 4688839a0e0STejun Heo } 4698839a0e0STejun Heo 4708839a0e0STejun Heo /* 471ae1b1539STejun Heo * @rq should go through flush machinery. Mark it part of flush 472ae1b1539STejun Heo * sequence and submit for further processing. 4738839a0e0STejun Heo */ 474ae1b1539STejun Heo memset(&rq->flush, 0, sizeof(rq->flush)); 475ae1b1539STejun Heo INIT_LIST_HEAD(&rq->flush.list); 476e8064021SChristoph Hellwig rq->rq_flags |= RQF_FLUSH_SEQ; 4774853abaaSJeff Moyer rq->flush.saved_end_io = rq->end_io; /* Usually NULL */ 478320ae51fSJens Axboe if (q->mq_ops) { 479320ae51fSJens Axboe rq->end_io = mq_flush_data_end_io; 480320ae51fSJens Axboe 4817c94e1c1SMing Lei spin_lock_irq(&fq->mq_flush_lock); 4820bae352dSMing Lei blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); 4837c94e1c1SMing Lei spin_unlock_irq(&fq->mq_flush_lock); 484320ae51fSJens Axboe return; 485320ae51fSJens Axboe } 486ae1b1539STejun Heo rq->end_io = flush_data_end_io; 487ae1b1539STejun Heo 4880bae352dSMing Lei blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0); 489ae1b1539STejun Heo } 490ae1b1539STejun Heo 491ae1b1539STejun Heo /** 4928839a0e0STejun Heo * blkdev_issue_flush - queue a flush 4938839a0e0STejun Heo * @bdev: blockdev to issue flush for 4948839a0e0STejun Heo * @gfp_mask: memory allocation flags (for bio_alloc) 4958839a0e0STejun Heo * @error_sector: error sector 4968839a0e0STejun Heo * 4978839a0e0STejun Heo * Description: 4988839a0e0STejun Heo * Issue a flush for the block device in question. Caller can supply 4998839a0e0STejun Heo * room for storing the error offset in case of a flush error, if they 5001be7d207SEric Biggers * wish to. 5018839a0e0STejun Heo */ 5028839a0e0STejun Heo int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, 503dd3932edSChristoph Hellwig sector_t *error_sector) 5048839a0e0STejun Heo { 5058839a0e0STejun Heo struct request_queue *q; 5068839a0e0STejun Heo struct bio *bio; 5078839a0e0STejun Heo int ret = 0; 5088839a0e0STejun Heo 5098839a0e0STejun Heo if (bdev->bd_disk == NULL) 5108839a0e0STejun Heo return -ENXIO; 5118839a0e0STejun Heo 5128839a0e0STejun Heo q = bdev_get_queue(bdev); 5138839a0e0STejun Heo if (!q) 5148839a0e0STejun Heo return -ENXIO; 5158839a0e0STejun Heo 5168839a0e0STejun Heo /* 5178839a0e0STejun Heo * some block devices may not have their queue correctly set up here 5188839a0e0STejun Heo * (e.g. loop device without a backing file) and so issuing a flush 5198839a0e0STejun Heo * here will panic. Ensure there is a request function before issuing 520d391a2ddSTejun Heo * the flush. 5218839a0e0STejun Heo */ 5228839a0e0STejun Heo if (!q->make_request_fn) 5238839a0e0STejun Heo return -ENXIO; 5248839a0e0STejun Heo 5258839a0e0STejun Heo bio = bio_alloc(gfp_mask, 0); 5268839a0e0STejun Heo bio->bi_bdev = bdev; 52770fd7614SChristoph Hellwig bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; 5288839a0e0STejun Heo 5294e49ea4aSMike Christie ret = submit_bio_wait(bio); 530dd3932edSChristoph Hellwig 5318839a0e0STejun Heo /* 5328839a0e0STejun Heo * The driver must store the error location in ->bi_sector, if 5338839a0e0STejun Heo * it supports it. For non-stacked drivers, this should be 5348839a0e0STejun Heo * copied from blk_rq_pos(rq). 5358839a0e0STejun Heo */ 5368839a0e0STejun Heo if (error_sector) 5374f024f37SKent Overstreet *error_sector = bio->bi_iter.bi_sector; 5388839a0e0STejun Heo 5398839a0e0STejun Heo bio_put(bio); 5408839a0e0STejun Heo return ret; 5418839a0e0STejun Heo } 5428839a0e0STejun Heo EXPORT_SYMBOL(blkdev_issue_flush); 543320ae51fSJens Axboe 544f70ced09SMing Lei struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q, 545f70ced09SMing Lei int node, int cmd_size) 546320ae51fSJens Axboe { 5477c94e1c1SMing Lei struct blk_flush_queue *fq; 5487c94e1c1SMing Lei int rq_sz = sizeof(struct request); 5491bcb1eadSMing Lei 550f70ced09SMing Lei fq = kzalloc_node(sizeof(*fq), GFP_KERNEL, node); 5517c94e1c1SMing Lei if (!fq) 5527c94e1c1SMing Lei goto fail; 5531bcb1eadSMing Lei 5546d247d7fSChristoph Hellwig if (q->mq_ops) 5557c94e1c1SMing Lei spin_lock_init(&fq->mq_flush_lock); 5567c94e1c1SMing Lei 5576d247d7fSChristoph Hellwig rq_sz = round_up(rq_sz + cmd_size, cache_line_size()); 558f70ced09SMing Lei fq->flush_rq = kzalloc_node(rq_sz, GFP_KERNEL, node); 5597c94e1c1SMing Lei if (!fq->flush_rq) 5607c94e1c1SMing Lei goto fail_rq; 5617c94e1c1SMing Lei 5627c94e1c1SMing Lei INIT_LIST_HEAD(&fq->flush_queue[0]); 5637c94e1c1SMing Lei INIT_LIST_HEAD(&fq->flush_queue[1]); 5647c94e1c1SMing Lei INIT_LIST_HEAD(&fq->flush_data_in_flight); 5657c94e1c1SMing Lei 5667c94e1c1SMing Lei return fq; 5677c94e1c1SMing Lei 5687c94e1c1SMing Lei fail_rq: 5697c94e1c1SMing Lei kfree(fq); 5707c94e1c1SMing Lei fail: 5717c94e1c1SMing Lei return NULL; 5727c94e1c1SMing Lei } 5737c94e1c1SMing Lei 574ba483388SMing Lei void blk_free_flush_queue(struct blk_flush_queue *fq) 5757c94e1c1SMing Lei { 5767c94e1c1SMing Lei /* bio based request queue hasn't flush queue */ 5777c94e1c1SMing Lei if (!fq) 5787c94e1c1SMing Lei return; 5797c94e1c1SMing Lei 5807c94e1c1SMing Lei kfree(fq->flush_rq); 5817c94e1c1SMing Lei kfree(fq); 582320ae51fSJens Axboe } 583