1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2022-2023 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <djwong@kernel.org> 5 */ 6 #include "xfs.h" 7 #include "xfs_fs.h" 8 #include "xfs_shared.h" 9 #include "xfs_format.h" 10 #include "xfs_trans_resv.h" 11 #include "xfs_mount.h" 12 #include "xfs_ag.h" 13 #include "xfs_trace.h" 14 15 /* 16 * Use a static key here to reduce the overhead of xfs_drain_rele. If the 17 * compiler supports jump labels, the static branch will be replaced by a nop 18 * sled when there are no xfs_drain_wait callers. Online fsck is currently 19 * the only caller, so this is a reasonable tradeoff. 20 * 21 * Note: Patching the kernel code requires taking the cpu hotplug lock. Other 22 * parts of the kernel allocate memory with that lock held, which means that 23 * XFS callers cannot hold any locks that might be used by memory reclaim or 24 * writeback when calling the static_branch_{inc,dec} functions. 25 */ 26 static DEFINE_STATIC_KEY_FALSE(xfs_drain_waiter_gate); 27 28 void 29 xfs_drain_wait_disable(void) 30 { 31 static_branch_dec(&xfs_drain_waiter_gate); 32 } 33 34 void 35 xfs_drain_wait_enable(void) 36 { 37 static_branch_inc(&xfs_drain_waiter_gate); 38 } 39 40 void 41 xfs_defer_drain_init( 42 struct xfs_defer_drain *dr) 43 { 44 atomic_set(&dr->dr_count, 0); 45 init_waitqueue_head(&dr->dr_waiters); 46 } 47 48 void 49 xfs_defer_drain_free(struct xfs_defer_drain *dr) 50 { 51 ASSERT(atomic_read(&dr->dr_count) == 0); 52 } 53 54 /* Increase the pending intent count. */ 55 static inline void xfs_defer_drain_grab(struct xfs_defer_drain *dr) 56 { 57 atomic_inc(&dr->dr_count); 58 } 59 60 static inline bool has_waiters(struct wait_queue_head *wq_head) 61 { 62 /* 63 * This memory barrier is paired with the one in set_current_state on 64 * the waiting side. 65 */ 66 smp_mb__after_atomic(); 67 return waitqueue_active(wq_head); 68 } 69 70 /* Decrease the pending intent count, and wake any waiters, if appropriate. */ 71 static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr) 72 { 73 if (atomic_dec_and_test(&dr->dr_count) && 74 static_branch_unlikely(&xfs_drain_waiter_gate) && 75 has_waiters(&dr->dr_waiters)) 76 wake_up(&dr->dr_waiters); 77 } 78 79 /* Are there intents pending? */ 80 static inline bool xfs_defer_drain_busy(struct xfs_defer_drain *dr) 81 { 82 return atomic_read(&dr->dr_count) > 0; 83 } 84 85 /* 86 * Wait for the pending intent count for a drain to hit zero. 87 * 88 * Callers must not hold any locks that would prevent intents from being 89 * finished. 90 */ 91 static inline int xfs_defer_drain_wait(struct xfs_defer_drain *dr) 92 { 93 return wait_event_killable(dr->dr_waiters, !xfs_defer_drain_busy(dr)); 94 } 95 96 /* 97 * Get a passive reference to an AG and declare an intent to update its 98 * metadata. 99 */ 100 struct xfs_perag * 101 xfs_perag_intent_get( 102 struct xfs_mount *mp, 103 xfs_agnumber_t agno) 104 { 105 struct xfs_perag *pag; 106 107 pag = xfs_perag_get(mp, agno); 108 if (!pag) 109 return NULL; 110 111 xfs_perag_intent_hold(pag); 112 return pag; 113 } 114 115 /* 116 * Release our intent to update this AG's metadata, and then release our 117 * passive ref to the AG. 118 */ 119 void 120 xfs_perag_intent_put( 121 struct xfs_perag *pag) 122 { 123 xfs_perag_intent_rele(pag); 124 xfs_perag_put(pag); 125 } 126 127 /* 128 * Declare an intent to update AG metadata. Other threads that need exclusive 129 * access can decide to back off if they see declared intentions. 130 */ 131 void 132 xfs_perag_intent_hold( 133 struct xfs_perag *pag) 134 { 135 trace_xfs_perag_intent_hold(pag, __return_address); 136 xfs_defer_drain_grab(&pag->pag_intents_drain); 137 } 138 139 /* Release our intent to update this AG's metadata. */ 140 void 141 xfs_perag_intent_rele( 142 struct xfs_perag *pag) 143 { 144 trace_xfs_perag_intent_rele(pag, __return_address); 145 xfs_defer_drain_rele(&pag->pag_intents_drain); 146 } 147 148 /* 149 * Wait for the intent update count for this AG to hit zero. 150 * Callers must not hold any AG header buffers. 151 */ 152 int 153 xfs_perag_intent_drain( 154 struct xfs_perag *pag) 155 { 156 trace_xfs_perag_wait_intents(pag, __return_address); 157 return xfs_defer_drain_wait(&pag->pag_intents_drain); 158 } 159 160 /* Has anyone declared an intent to update this AG? */ 161 bool 162 xfs_perag_intent_busy( 163 struct xfs_perag *pag) 164 { 165 return xfs_defer_drain_busy(&pag->pag_intents_drain); 166 } 167