1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2008 Oracle. All rights reserved. 4 */ 5 6 #include <linux/sched.h> 7 #include <linux/pagemap.h> 8 #include <linux/spinlock.h> 9 #include <linux/page-flags.h> 10 #include <asm/bug.h> 11 #include "ctree.h" 12 #include "extent_io.h" 13 #include "locking.h" 14 15 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb); 16 17 /* 18 * if we currently have a spinning reader or writer lock 19 * (indicated by the rw flag) this will bump the count 20 * of blocking holders and drop the spinlock. 21 */ 22 void btrfs_set_lock_blocking_rw(struct extent_buffer *eb, int rw) 23 { 24 /* 25 * no lock is required. The lock owner may change if 26 * we have a read lock, but it won't change to or away 27 * from us. If we have the write lock, we are the owner 28 * and it'll never change. 29 */ 30 if (eb->lock_nested && current->pid == eb->lock_owner) 31 return; 32 if (rw == BTRFS_WRITE_LOCK) { 33 if (atomic_read(&eb->blocking_writers) == 0) { 34 WARN_ON(atomic_read(&eb->spinning_writers) != 1); 35 atomic_dec(&eb->spinning_writers); 36 btrfs_assert_tree_locked(eb); 37 atomic_inc(&eb->blocking_writers); 38 write_unlock(&eb->lock); 39 } 40 } else if (rw == BTRFS_READ_LOCK) { 41 btrfs_assert_tree_read_locked(eb); 42 atomic_inc(&eb->blocking_readers); 43 WARN_ON(atomic_read(&eb->spinning_readers) == 0); 44 atomic_dec(&eb->spinning_readers); 45 read_unlock(&eb->lock); 46 } 47 } 48 49 /* 50 * if we currently have a blocking lock, take the spinlock 51 * and drop our blocking count 52 */ 53 void btrfs_clear_lock_blocking_rw(struct extent_buffer *eb, int rw) 54 { 55 /* 56 * no lock is required. The lock owner may change if 57 * we have a read lock, but it won't change to or away 58 * from us. If we have the write lock, we are the owner 59 * and it'll never change. 60 */ 61 if (eb->lock_nested && current->pid == eb->lock_owner) 62 return; 63 64 if (rw == BTRFS_WRITE_LOCK_BLOCKING) { 65 BUG_ON(atomic_read(&eb->blocking_writers) != 1); 66 write_lock(&eb->lock); 67 WARN_ON(atomic_read(&eb->spinning_writers)); 68 atomic_inc(&eb->spinning_writers); 69 /* 70 * atomic_dec_and_test implies a barrier for waitqueue_active 71 */ 72 if (atomic_dec_and_test(&eb->blocking_writers) && 73 waitqueue_active(&eb->write_lock_wq)) 74 wake_up(&eb->write_lock_wq); 75 } else if (rw == BTRFS_READ_LOCK_BLOCKING) { 76 BUG_ON(atomic_read(&eb->blocking_readers) == 0); 77 read_lock(&eb->lock); 78 atomic_inc(&eb->spinning_readers); 79 /* 80 * atomic_dec_and_test implies a barrier for waitqueue_active 81 */ 82 if (atomic_dec_and_test(&eb->blocking_readers) && 83 waitqueue_active(&eb->read_lock_wq)) 84 wake_up(&eb->read_lock_wq); 85 } 86 } 87 88 /* 89 * take a spinning read lock. This will wait for any blocking 90 * writers 91 */ 92 void btrfs_tree_read_lock(struct extent_buffer *eb) 93 { 94 again: 95 BUG_ON(!atomic_read(&eb->blocking_writers) && 96 current->pid == eb->lock_owner); 97 98 read_lock(&eb->lock); 99 if (atomic_read(&eb->blocking_writers) && 100 current->pid == eb->lock_owner) { 101 /* 102 * This extent is already write-locked by our thread. We allow 103 * an additional read lock to be added because it's for the same 104 * thread. btrfs_find_all_roots() depends on this as it may be 105 * called on a partly (write-)locked tree. 106 */ 107 BUG_ON(eb->lock_nested); 108 eb->lock_nested = 1; 109 read_unlock(&eb->lock); 110 return; 111 } 112 if (atomic_read(&eb->blocking_writers)) { 113 read_unlock(&eb->lock); 114 wait_event(eb->write_lock_wq, 115 atomic_read(&eb->blocking_writers) == 0); 116 goto again; 117 } 118 atomic_inc(&eb->read_locks); 119 atomic_inc(&eb->spinning_readers); 120 } 121 122 /* 123 * take a spinning read lock. 124 * returns 1 if we get the read lock and 0 if we don't 125 * this won't wait for blocking writers 126 */ 127 int btrfs_tree_read_lock_atomic(struct extent_buffer *eb) 128 { 129 if (atomic_read(&eb->blocking_writers)) 130 return 0; 131 132 read_lock(&eb->lock); 133 if (atomic_read(&eb->blocking_writers)) { 134 read_unlock(&eb->lock); 135 return 0; 136 } 137 atomic_inc(&eb->read_locks); 138 atomic_inc(&eb->spinning_readers); 139 return 1; 140 } 141 142 /* 143 * returns 1 if we get the read lock and 0 if we don't 144 * this won't wait for blocking writers 145 */ 146 int btrfs_try_tree_read_lock(struct extent_buffer *eb) 147 { 148 if (atomic_read(&eb->blocking_writers)) 149 return 0; 150 151 if (!read_trylock(&eb->lock)) 152 return 0; 153 154 if (atomic_read(&eb->blocking_writers)) { 155 read_unlock(&eb->lock); 156 return 0; 157 } 158 atomic_inc(&eb->read_locks); 159 atomic_inc(&eb->spinning_readers); 160 return 1; 161 } 162 163 /* 164 * returns 1 if we get the read lock and 0 if we don't 165 * this won't wait for blocking writers or readers 166 */ 167 int btrfs_try_tree_write_lock(struct extent_buffer *eb) 168 { 169 if (atomic_read(&eb->blocking_writers) || 170 atomic_read(&eb->blocking_readers)) 171 return 0; 172 173 write_lock(&eb->lock); 174 if (atomic_read(&eb->blocking_writers) || 175 atomic_read(&eb->blocking_readers)) { 176 write_unlock(&eb->lock); 177 return 0; 178 } 179 atomic_inc(&eb->write_locks); 180 atomic_inc(&eb->spinning_writers); 181 eb->lock_owner = current->pid; 182 return 1; 183 } 184 185 /* 186 * drop a spinning read lock 187 */ 188 void btrfs_tree_read_unlock(struct extent_buffer *eb) 189 { 190 /* 191 * if we're nested, we have the write lock. No new locking 192 * is needed as long as we are the lock owner. 193 * The write unlock will do a barrier for us, and the lock_nested 194 * field only matters to the lock owner. 195 */ 196 if (eb->lock_nested && current->pid == eb->lock_owner) { 197 eb->lock_nested = 0; 198 return; 199 } 200 btrfs_assert_tree_read_locked(eb); 201 WARN_ON(atomic_read(&eb->spinning_readers) == 0); 202 atomic_dec(&eb->spinning_readers); 203 atomic_dec(&eb->read_locks); 204 read_unlock(&eb->lock); 205 } 206 207 /* 208 * drop a blocking read lock 209 */ 210 void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb) 211 { 212 /* 213 * if we're nested, we have the write lock. No new locking 214 * is needed as long as we are the lock owner. 215 * The write unlock will do a barrier for us, and the lock_nested 216 * field only matters to the lock owner. 217 */ 218 if (eb->lock_nested && current->pid == eb->lock_owner) { 219 eb->lock_nested = 0; 220 return; 221 } 222 btrfs_assert_tree_read_locked(eb); 223 WARN_ON(atomic_read(&eb->blocking_readers) == 0); 224 /* 225 * atomic_dec_and_test implies a barrier for waitqueue_active 226 */ 227 if (atomic_dec_and_test(&eb->blocking_readers) && 228 waitqueue_active(&eb->read_lock_wq)) 229 wake_up(&eb->read_lock_wq); 230 atomic_dec(&eb->read_locks); 231 } 232 233 /* 234 * take a spinning write lock. This will wait for both 235 * blocking readers or writers 236 */ 237 void btrfs_tree_lock(struct extent_buffer *eb) 238 { 239 WARN_ON(eb->lock_owner == current->pid); 240 again: 241 wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0); 242 wait_event(eb->write_lock_wq, atomic_read(&eb->blocking_writers) == 0); 243 write_lock(&eb->lock); 244 if (atomic_read(&eb->blocking_readers)) { 245 write_unlock(&eb->lock); 246 wait_event(eb->read_lock_wq, 247 atomic_read(&eb->blocking_readers) == 0); 248 goto again; 249 } 250 if (atomic_read(&eb->blocking_writers)) { 251 write_unlock(&eb->lock); 252 wait_event(eb->write_lock_wq, 253 atomic_read(&eb->blocking_writers) == 0); 254 goto again; 255 } 256 WARN_ON(atomic_read(&eb->spinning_writers)); 257 atomic_inc(&eb->spinning_writers); 258 atomic_inc(&eb->write_locks); 259 eb->lock_owner = current->pid; 260 } 261 262 /* 263 * drop a spinning or a blocking write lock. 264 */ 265 void btrfs_tree_unlock(struct extent_buffer *eb) 266 { 267 int blockers = atomic_read(&eb->blocking_writers); 268 269 BUG_ON(blockers > 1); 270 271 btrfs_assert_tree_locked(eb); 272 eb->lock_owner = 0; 273 atomic_dec(&eb->write_locks); 274 275 if (blockers) { 276 WARN_ON(atomic_read(&eb->spinning_writers)); 277 atomic_dec(&eb->blocking_writers); 278 /* 279 * Make sure counter is updated before we wake up waiters. 280 */ 281 smp_mb__after_atomic(); 282 if (waitqueue_active(&eb->write_lock_wq)) 283 wake_up(&eb->write_lock_wq); 284 } else { 285 WARN_ON(atomic_read(&eb->spinning_writers) != 1); 286 atomic_dec(&eb->spinning_writers); 287 write_unlock(&eb->lock); 288 } 289 } 290 291 void btrfs_assert_tree_locked(struct extent_buffer *eb) 292 { 293 BUG_ON(!atomic_read(&eb->write_locks)); 294 } 295 296 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) 297 { 298 BUG_ON(!atomic_read(&eb->read_locks)); 299 } 300