1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. 4 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/sched.h> 10 #include <linux/slab.h> 11 #include <linux/spinlock.h> 12 #include <linux/buffer_head.h> 13 #include <linux/delay.h> 14 #include <linux/sort.h> 15 #include <linux/hash.h> 16 #include <linux/jhash.h> 17 #include <linux/kallsyms.h> 18 #include <linux/gfs2_ondisk.h> 19 #include <linux/list.h> 20 #include <linux/wait.h> 21 #include <linux/module.h> 22 #include <linux/uaccess.h> 23 #include <linux/seq_file.h> 24 #include <linux/debugfs.h> 25 #include <linux/kthread.h> 26 #include <linux/freezer.h> 27 #include <linux/workqueue.h> 28 #include <linux/jiffies.h> 29 #include <linux/rcupdate.h> 30 #include <linux/rculist_bl.h> 31 #include <linux/bit_spinlock.h> 32 #include <linux/percpu.h> 33 #include <linux/list_sort.h> 34 #include <linux/lockref.h> 35 #include <linux/rhashtable.h> 36 37 #include "gfs2.h" 38 #include "incore.h" 39 #include "glock.h" 40 #include "glops.h" 41 #include "inode.h" 42 #include "lops.h" 43 #include "meta_io.h" 44 #include "quota.h" 45 #include "super.h" 46 #include "util.h" 47 #include "bmap.h" 48 #define CREATE_TRACE_POINTS 49 #include "trace_gfs2.h" 50 51 struct gfs2_glock_iter { 52 struct gfs2_sbd *sdp; /* incore superblock */ 53 struct rhashtable_iter hti; /* rhashtable iterator */ 54 struct gfs2_glock *gl; /* current glock struct */ 55 loff_t last_pos; /* last position */ 56 }; 57 58 typedef void (*glock_examiner) (struct gfs2_glock * gl); 59 60 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target); 61 62 static struct dentry *gfs2_root; 63 static struct workqueue_struct *glock_workqueue; 64 struct workqueue_struct *gfs2_delete_workqueue; 65 static LIST_HEAD(lru_list); 66 static atomic_t lru_count = ATOMIC_INIT(0); 67 static DEFINE_SPINLOCK(lru_lock); 68 69 #define GFS2_GL_HASH_SHIFT 15 70 #define GFS2_GL_HASH_SIZE BIT(GFS2_GL_HASH_SHIFT) 71 72 static const struct rhashtable_params ht_parms = { 73 .nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4, 74 .key_len = offsetofend(struct lm_lockname, ln_type), 75 .key_offset = offsetof(struct gfs2_glock, gl_name), 76 .head_offset = offsetof(struct gfs2_glock, gl_node), 77 }; 78 79 static struct rhashtable gl_hash_table; 80 81 #define GLOCK_WAIT_TABLE_BITS 12 82 #define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS) 83 static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned; 84 85 struct wait_glock_queue { 86 struct lm_lockname *name; 87 wait_queue_entry_t wait; 88 }; 89 90 static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode, 91 int sync, void *key) 92 { 93 struct wait_glock_queue *wait_glock = 94 container_of(wait, struct wait_glock_queue, wait); 95 struct lm_lockname *wait_name = wait_glock->name; 96 struct lm_lockname *wake_name = key; 97 98 if (wake_name->ln_sbd != wait_name->ln_sbd || 99 wake_name->ln_number != wait_name->ln_number || 100 wake_name->ln_type != wait_name->ln_type) 101 return 0; 102 return autoremove_wake_function(wait, mode, sync, key); 103 } 104 105 static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name) 106 { 107 u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0); 108 109 return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS); 110 } 111 112 /** 113 * wake_up_glock - Wake up waiters on a glock 114 * @gl: the glock 115 */ 116 static void wake_up_glock(struct gfs2_glock *gl) 117 { 118 wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name); 119 120 if (waitqueue_active(wq)) 121 __wake_up(wq, TASK_NORMAL, 1, &gl->gl_name); 122 } 123 124 static void gfs2_glock_dealloc(struct rcu_head *rcu) 125 { 126 struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu); 127 128 if (gl->gl_ops->go_flags & GLOF_ASPACE) { 129 kmem_cache_free(gfs2_glock_aspace_cachep, gl); 130 } else { 131 kfree(gl->gl_lksb.sb_lvbptr); 132 kmem_cache_free(gfs2_glock_cachep, gl); 133 } 134 } 135 136 void gfs2_glock_free(struct gfs2_glock *gl) 137 { 138 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; 139 140 BUG_ON(atomic_read(&gl->gl_revokes)); 141 rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms); 142 smp_mb(); 143 wake_up_glock(gl); 144 call_rcu(&gl->gl_rcu, gfs2_glock_dealloc); 145 if (atomic_dec_and_test(&sdp->sd_glock_disposal)) 146 wake_up(&sdp->sd_glock_wait); 147 } 148 149 /** 150 * gfs2_glock_hold() - increment reference count on glock 151 * @gl: The glock to hold 152 * 153 */ 154 155 void gfs2_glock_hold(struct gfs2_glock *gl) 156 { 157 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref)); 158 lockref_get(&gl->gl_lockref); 159 } 160 161 /** 162 * demote_ok - Check to see if it's ok to unlock a glock 163 * @gl: the glock 164 * 165 * Returns: 1 if it's ok 166 */ 167 168 static int demote_ok(const struct gfs2_glock *gl) 169 { 170 const struct gfs2_glock_operations *glops = gl->gl_ops; 171 172 if (gl->gl_state == LM_ST_UNLOCKED) 173 return 0; 174 if (!list_empty(&gl->gl_holders)) 175 return 0; 176 if (glops->go_demote_ok) 177 return glops->go_demote_ok(gl); 178 return 1; 179 } 180 181 182 void gfs2_glock_add_to_lru(struct gfs2_glock *gl) 183 { 184 if (!(gl->gl_ops->go_flags & GLOF_LRU)) 185 return; 186 187 spin_lock(&lru_lock); 188 189 list_del(&gl->gl_lru); 190 list_add_tail(&gl->gl_lru, &lru_list); 191 192 if (!test_bit(GLF_LRU, &gl->gl_flags)) { 193 set_bit(GLF_LRU, &gl->gl_flags); 194 atomic_inc(&lru_count); 195 } 196 197 spin_unlock(&lru_lock); 198 } 199 200 static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl) 201 { 202 if (!(gl->gl_ops->go_flags & GLOF_LRU)) 203 return; 204 205 spin_lock(&lru_lock); 206 if (test_bit(GLF_LRU, &gl->gl_flags)) { 207 list_del_init(&gl->gl_lru); 208 atomic_dec(&lru_count); 209 clear_bit(GLF_LRU, &gl->gl_flags); 210 } 211 spin_unlock(&lru_lock); 212 } 213 214 /* 215 * Enqueue the glock on the work queue. Passes one glock reference on to the 216 * work queue. 217 */ 218 static void __gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) { 219 if (!queue_delayed_work(glock_workqueue, &gl->gl_work, delay)) { 220 /* 221 * We are holding the lockref spinlock, and the work was still 222 * queued above. The queued work (glock_work_func) takes that 223 * spinlock before dropping its glock reference(s), so it 224 * cannot have dropped them in the meantime. 225 */ 226 GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2); 227 gl->gl_lockref.count--; 228 } 229 } 230 231 static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) { 232 spin_lock(&gl->gl_lockref.lock); 233 __gfs2_glock_queue_work(gl, delay); 234 spin_unlock(&gl->gl_lockref.lock); 235 } 236 237 static void __gfs2_glock_put(struct gfs2_glock *gl) 238 { 239 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; 240 struct address_space *mapping = gfs2_glock2aspace(gl); 241 242 lockref_mark_dead(&gl->gl_lockref); 243 244 gfs2_glock_remove_from_lru(gl); 245 spin_unlock(&gl->gl_lockref.lock); 246 GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders)); 247 GLOCK_BUG_ON(gl, mapping && mapping->nrpages); 248 trace_gfs2_glock_put(gl); 249 sdp->sd_lockstruct.ls_ops->lm_put_lock(gl); 250 } 251 252 /* 253 * Cause the glock to be put in work queue context. 254 */ 255 void gfs2_glock_queue_put(struct gfs2_glock *gl) 256 { 257 gfs2_glock_queue_work(gl, 0); 258 } 259 260 /** 261 * gfs2_glock_put() - Decrement reference count on glock 262 * @gl: The glock to put 263 * 264 */ 265 266 void gfs2_glock_put(struct gfs2_glock *gl) 267 { 268 if (lockref_put_or_lock(&gl->gl_lockref)) 269 return; 270 271 __gfs2_glock_put(gl); 272 } 273 274 /** 275 * may_grant - check if its ok to grant a new lock 276 * @gl: The glock 277 * @gh: The lock request which we wish to grant 278 * 279 * Returns: true if its ok to grant the lock 280 */ 281 282 static inline int may_grant(const struct gfs2_glock *gl, const struct gfs2_holder *gh) 283 { 284 const struct gfs2_holder *gh_head = list_entry(gl->gl_holders.next, const struct gfs2_holder, gh_list); 285 if ((gh->gh_state == LM_ST_EXCLUSIVE || 286 gh_head->gh_state == LM_ST_EXCLUSIVE) && gh != gh_head) 287 return 0; 288 if (gl->gl_state == gh->gh_state) 289 return 1; 290 if (gh->gh_flags & GL_EXACT) 291 return 0; 292 if (gl->gl_state == LM_ST_EXCLUSIVE) { 293 if (gh->gh_state == LM_ST_SHARED && gh_head->gh_state == LM_ST_SHARED) 294 return 1; 295 if (gh->gh_state == LM_ST_DEFERRED && gh_head->gh_state == LM_ST_DEFERRED) 296 return 1; 297 } 298 if (gl->gl_state != LM_ST_UNLOCKED && (gh->gh_flags & LM_FLAG_ANY)) 299 return 1; 300 return 0; 301 } 302 303 static void gfs2_holder_wake(struct gfs2_holder *gh) 304 { 305 clear_bit(HIF_WAIT, &gh->gh_iflags); 306 smp_mb__after_atomic(); 307 wake_up_bit(&gh->gh_iflags, HIF_WAIT); 308 if (gh->gh_flags & GL_ASYNC) { 309 struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd; 310 311 wake_up(&sdp->sd_async_glock_wait); 312 } 313 } 314 315 /** 316 * do_error - Something unexpected has happened during a lock request 317 * 318 */ 319 320 static void do_error(struct gfs2_glock *gl, const int ret) 321 { 322 struct gfs2_holder *gh, *tmp; 323 324 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) { 325 if (test_bit(HIF_HOLDER, &gh->gh_iflags)) 326 continue; 327 if (ret & LM_OUT_ERROR) 328 gh->gh_error = -EIO; 329 else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) 330 gh->gh_error = GLR_TRYFAILED; 331 else 332 continue; 333 list_del_init(&gh->gh_list); 334 trace_gfs2_glock_queue(gh, 0); 335 gfs2_holder_wake(gh); 336 } 337 } 338 339 /** 340 * do_promote - promote as many requests as possible on the current queue 341 * @gl: The glock 342 * 343 * Returns: 1 if there is a blocked holder at the head of the list, or 2 344 * if a type specific operation is underway. 345 */ 346 347 static int do_promote(struct gfs2_glock *gl) 348 __releases(&gl->gl_lockref.lock) 349 __acquires(&gl->gl_lockref.lock) 350 { 351 const struct gfs2_glock_operations *glops = gl->gl_ops; 352 struct gfs2_holder *gh, *tmp; 353 int ret; 354 355 restart: 356 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) { 357 if (test_bit(HIF_HOLDER, &gh->gh_iflags)) 358 continue; 359 if (may_grant(gl, gh)) { 360 if (gh->gh_list.prev == &gl->gl_holders && 361 glops->go_lock) { 362 spin_unlock(&gl->gl_lockref.lock); 363 /* FIXME: eliminate this eventually */ 364 ret = glops->go_lock(gh); 365 spin_lock(&gl->gl_lockref.lock); 366 if (ret) { 367 if (ret == 1) 368 return 2; 369 gh->gh_error = ret; 370 list_del_init(&gh->gh_list); 371 trace_gfs2_glock_queue(gh, 0); 372 gfs2_holder_wake(gh); 373 goto restart; 374 } 375 set_bit(HIF_HOLDER, &gh->gh_iflags); 376 trace_gfs2_promote(gh, 1); 377 gfs2_holder_wake(gh); 378 goto restart; 379 } 380 set_bit(HIF_HOLDER, &gh->gh_iflags); 381 trace_gfs2_promote(gh, 0); 382 gfs2_holder_wake(gh); 383 continue; 384 } 385 if (gh->gh_list.prev == &gl->gl_holders) 386 return 1; 387 do_error(gl, 0); 388 break; 389 } 390 return 0; 391 } 392 393 /** 394 * find_first_waiter - find the first gh that's waiting for the glock 395 * @gl: the glock 396 */ 397 398 static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl) 399 { 400 struct gfs2_holder *gh; 401 402 list_for_each_entry(gh, &gl->gl_holders, gh_list) { 403 if (!test_bit(HIF_HOLDER, &gh->gh_iflags)) 404 return gh; 405 } 406 return NULL; 407 } 408 409 /** 410 * state_change - record that the glock is now in a different state 411 * @gl: the glock 412 * @new_state the new state 413 * 414 */ 415 416 static void state_change(struct gfs2_glock *gl, unsigned int new_state) 417 { 418 int held1, held2; 419 420 held1 = (gl->gl_state != LM_ST_UNLOCKED); 421 held2 = (new_state != LM_ST_UNLOCKED); 422 423 if (held1 != held2) { 424 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref)); 425 if (held2) 426 gl->gl_lockref.count++; 427 else 428 gl->gl_lockref.count--; 429 } 430 if (held1 && held2 && list_empty(&gl->gl_holders)) 431 clear_bit(GLF_QUEUED, &gl->gl_flags); 432 433 if (new_state != gl->gl_target) 434 /* shorten our minimum hold time */ 435 gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR, 436 GL_GLOCK_MIN_HOLD); 437 gl->gl_state = new_state; 438 gl->gl_tchange = jiffies; 439 } 440 441 static void gfs2_demote_wake(struct gfs2_glock *gl) 442 { 443 gl->gl_demote_state = LM_ST_EXCLUSIVE; 444 clear_bit(GLF_DEMOTE, &gl->gl_flags); 445 smp_mb__after_atomic(); 446 wake_up_bit(&gl->gl_flags, GLF_DEMOTE); 447 } 448 449 /** 450 * finish_xmote - The DLM has replied to one of our lock requests 451 * @gl: The glock 452 * @ret: The status from the DLM 453 * 454 */ 455 456 static void finish_xmote(struct gfs2_glock *gl, unsigned int ret) 457 { 458 const struct gfs2_glock_operations *glops = gl->gl_ops; 459 struct gfs2_holder *gh; 460 unsigned state = ret & LM_OUT_ST_MASK; 461 int rv; 462 463 spin_lock(&gl->gl_lockref.lock); 464 trace_gfs2_glock_state_change(gl, state); 465 state_change(gl, state); 466 gh = find_first_waiter(gl); 467 468 /* Demote to UN request arrived during demote to SH or DF */ 469 if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) && 470 state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED) 471 gl->gl_target = LM_ST_UNLOCKED; 472 473 /* Check for state != intended state */ 474 if (unlikely(state != gl->gl_target)) { 475 if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) { 476 /* move to back of queue and try next entry */ 477 if (ret & LM_OUT_CANCELED) { 478 if ((gh->gh_flags & LM_FLAG_PRIORITY) == 0) 479 list_move_tail(&gh->gh_list, &gl->gl_holders); 480 gh = find_first_waiter(gl); 481 gl->gl_target = gh->gh_state; 482 goto retry; 483 } 484 /* Some error or failed "try lock" - report it */ 485 if ((ret & LM_OUT_ERROR) || 486 (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) { 487 gl->gl_target = gl->gl_state; 488 do_error(gl, ret); 489 goto out; 490 } 491 } 492 switch(state) { 493 /* Unlocked due to conversion deadlock, try again */ 494 case LM_ST_UNLOCKED: 495 retry: 496 do_xmote(gl, gh, gl->gl_target); 497 break; 498 /* Conversion fails, unlock and try again */ 499 case LM_ST_SHARED: 500 case LM_ST_DEFERRED: 501 do_xmote(gl, gh, LM_ST_UNLOCKED); 502 break; 503 default: /* Everything else */ 504 fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n", 505 gl->gl_target, state); 506 GLOCK_BUG_ON(gl, 1); 507 } 508 spin_unlock(&gl->gl_lockref.lock); 509 return; 510 } 511 512 /* Fast path - we got what we asked for */ 513 if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) 514 gfs2_demote_wake(gl); 515 if (state != LM_ST_UNLOCKED) { 516 if (glops->go_xmote_bh) { 517 spin_unlock(&gl->gl_lockref.lock); 518 rv = glops->go_xmote_bh(gl, gh); 519 spin_lock(&gl->gl_lockref.lock); 520 if (rv) { 521 do_error(gl, rv); 522 goto out; 523 } 524 } 525 rv = do_promote(gl); 526 if (rv == 2) 527 goto out_locked; 528 } 529 out: 530 clear_bit(GLF_LOCK, &gl->gl_flags); 531 out_locked: 532 spin_unlock(&gl->gl_lockref.lock); 533 } 534 535 /** 536 * do_xmote - Calls the DLM to change the state of a lock 537 * @gl: The lock state 538 * @gh: The holder (only for promotes) 539 * @target: The target lock state 540 * 541 */ 542 543 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target) 544 __releases(&gl->gl_lockref.lock) 545 __acquires(&gl->gl_lockref.lock) 546 { 547 const struct gfs2_glock_operations *glops = gl->gl_ops; 548 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; 549 unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0); 550 int ret; 551 552 if (unlikely(gfs2_withdrawn(sdp)) && 553 target != LM_ST_UNLOCKED) 554 return; 555 lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP | 556 LM_FLAG_PRIORITY); 557 GLOCK_BUG_ON(gl, gl->gl_state == target); 558 GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target); 559 if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) && 560 glops->go_inval) { 561 /* 562 * If another process is already doing the invalidate, let that 563 * finish first. The glock state machine will get back to this 564 * holder again later. 565 */ 566 if (test_and_set_bit(GLF_INVALIDATE_IN_PROGRESS, 567 &gl->gl_flags)) 568 return; 569 do_error(gl, 0); /* Fail queued try locks */ 570 } 571 gl->gl_req = target; 572 set_bit(GLF_BLOCKING, &gl->gl_flags); 573 if ((gl->gl_req == LM_ST_UNLOCKED) || 574 (gl->gl_state == LM_ST_EXCLUSIVE) || 575 (lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB))) 576 clear_bit(GLF_BLOCKING, &gl->gl_flags); 577 spin_unlock(&gl->gl_lockref.lock); 578 if (glops->go_sync) 579 glops->go_sync(gl); 580 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) 581 glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA); 582 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags); 583 584 gfs2_glock_hold(gl); 585 if (sdp->sd_lockstruct.ls_ops->lm_lock) { 586 /* lock_dlm */ 587 ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags); 588 if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED && 589 target == LM_ST_UNLOCKED && 590 test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags)) { 591 finish_xmote(gl, target); 592 gfs2_glock_queue_work(gl, 0); 593 } 594 else if (ret) { 595 fs_err(sdp, "lm_lock ret %d\n", ret); 596 GLOCK_BUG_ON(gl, !gfs2_withdrawn(sdp)); 597 } 598 } else { /* lock_nolock */ 599 finish_xmote(gl, target); 600 gfs2_glock_queue_work(gl, 0); 601 } 602 603 spin_lock(&gl->gl_lockref.lock); 604 } 605 606 /** 607 * find_first_holder - find the first "holder" gh 608 * @gl: the glock 609 */ 610 611 static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl) 612 { 613 struct gfs2_holder *gh; 614 615 if (!list_empty(&gl->gl_holders)) { 616 gh = list_entry(gl->gl_holders.next, struct gfs2_holder, gh_list); 617 if (test_bit(HIF_HOLDER, &gh->gh_iflags)) 618 return gh; 619 } 620 return NULL; 621 } 622 623 /** 624 * run_queue - do all outstanding tasks related to a glock 625 * @gl: The glock in question 626 * @nonblock: True if we must not block in run_queue 627 * 628 */ 629 630 static void run_queue(struct gfs2_glock *gl, const int nonblock) 631 __releases(&gl->gl_lockref.lock) 632 __acquires(&gl->gl_lockref.lock) 633 { 634 struct gfs2_holder *gh = NULL; 635 int ret; 636 637 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) 638 return; 639 640 GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)); 641 642 if (test_bit(GLF_DEMOTE, &gl->gl_flags) && 643 gl->gl_demote_state != gl->gl_state) { 644 if (find_first_holder(gl)) 645 goto out_unlock; 646 if (nonblock) 647 goto out_sched; 648 set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags); 649 GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE); 650 gl->gl_target = gl->gl_demote_state; 651 } else { 652 if (test_bit(GLF_DEMOTE, &gl->gl_flags)) 653 gfs2_demote_wake(gl); 654 ret = do_promote(gl); 655 if (ret == 0) 656 goto out_unlock; 657 if (ret == 2) 658 goto out; 659 gh = find_first_waiter(gl); 660 gl->gl_target = gh->gh_state; 661 if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) 662 do_error(gl, 0); /* Fail queued try locks */ 663 } 664 do_xmote(gl, gh, gl->gl_target); 665 out: 666 return; 667 668 out_sched: 669 clear_bit(GLF_LOCK, &gl->gl_flags); 670 smp_mb__after_atomic(); 671 gl->gl_lockref.count++; 672 __gfs2_glock_queue_work(gl, 0); 673 return; 674 675 out_unlock: 676 clear_bit(GLF_LOCK, &gl->gl_flags); 677 smp_mb__after_atomic(); 678 return; 679 } 680 681 static void delete_work_func(struct work_struct *work) 682 { 683 struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_delete); 684 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; 685 struct inode *inode; 686 u64 no_addr = gl->gl_name.ln_number; 687 688 /* If someone's using this glock to create a new dinode, the block must 689 have been freed by another node, then re-used, in which case our 690 iopen callback is too late after the fact. Ignore it. */ 691 if (test_bit(GLF_INODE_CREATING, &gl->gl_flags)) 692 goto out; 693 694 inode = gfs2_lookup_by_inum(sdp, no_addr, NULL, GFS2_BLKST_UNLINKED); 695 if (!IS_ERR_OR_NULL(inode)) { 696 d_prune_aliases(inode); 697 iput(inode); 698 } 699 out: 700 gfs2_glock_put(gl); 701 } 702 703 static void glock_work_func(struct work_struct *work) 704 { 705 unsigned long delay = 0; 706 struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work); 707 unsigned int drop_refs = 1; 708 709 if (test_and_clear_bit(GLF_REPLY_PENDING, &gl->gl_flags)) { 710 finish_xmote(gl, gl->gl_reply); 711 drop_refs++; 712 } 713 spin_lock(&gl->gl_lockref.lock); 714 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) && 715 gl->gl_state != LM_ST_UNLOCKED && 716 gl->gl_demote_state != LM_ST_EXCLUSIVE) { 717 unsigned long holdtime, now = jiffies; 718 719 holdtime = gl->gl_tchange + gl->gl_hold_time; 720 if (time_before(now, holdtime)) 721 delay = holdtime - now; 722 723 if (!delay) { 724 clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags); 725 set_bit(GLF_DEMOTE, &gl->gl_flags); 726 } 727 } 728 run_queue(gl, 0); 729 if (delay) { 730 /* Keep one glock reference for the work we requeue. */ 731 drop_refs--; 732 if (gl->gl_name.ln_type != LM_TYPE_INODE) 733 delay = 0; 734 __gfs2_glock_queue_work(gl, delay); 735 } 736 737 /* 738 * Drop the remaining glock references manually here. (Mind that 739 * __gfs2_glock_queue_work depends on the lockref spinlock begin held 740 * here as well.) 741 */ 742 gl->gl_lockref.count -= drop_refs; 743 if (!gl->gl_lockref.count) { 744 __gfs2_glock_put(gl); 745 return; 746 } 747 spin_unlock(&gl->gl_lockref.lock); 748 } 749 750 static struct gfs2_glock *find_insert_glock(struct lm_lockname *name, 751 struct gfs2_glock *new) 752 { 753 struct wait_glock_queue wait; 754 wait_queue_head_t *wq = glock_waitqueue(name); 755 struct gfs2_glock *gl; 756 757 wait.name = name; 758 init_wait(&wait.wait); 759 wait.wait.func = glock_wake_function; 760 761 again: 762 prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE); 763 rcu_read_lock(); 764 if (new) { 765 gl = rhashtable_lookup_get_insert_fast(&gl_hash_table, 766 &new->gl_node, ht_parms); 767 if (IS_ERR(gl)) 768 goto out; 769 } else { 770 gl = rhashtable_lookup_fast(&gl_hash_table, 771 name, ht_parms); 772 } 773 if (gl && !lockref_get_not_dead(&gl->gl_lockref)) { 774 rcu_read_unlock(); 775 schedule(); 776 goto again; 777 } 778 out: 779 rcu_read_unlock(); 780 finish_wait(wq, &wait.wait); 781 return gl; 782 } 783 784 /** 785 * gfs2_glock_get() - Get a glock, or create one if one doesn't exist 786 * @sdp: The GFS2 superblock 787 * @number: the lock number 788 * @glops: The glock_operations to use 789 * @create: If 0, don't create the glock if it doesn't exist 790 * @glp: the glock is returned here 791 * 792 * This does not lock a glock, just finds/creates structures for one. 793 * 794 * Returns: errno 795 */ 796 797 int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number, 798 const struct gfs2_glock_operations *glops, int create, 799 struct gfs2_glock **glp) 800 { 801 struct super_block *s = sdp->sd_vfs; 802 struct lm_lockname name = { .ln_number = number, 803 .ln_type = glops->go_type, 804 .ln_sbd = sdp }; 805 struct gfs2_glock *gl, *tmp; 806 struct address_space *mapping; 807 struct kmem_cache *cachep; 808 int ret = 0; 809 810 gl = find_insert_glock(&name, NULL); 811 if (gl) { 812 *glp = gl; 813 return 0; 814 } 815 if (!create) 816 return -ENOENT; 817 818 if (glops->go_flags & GLOF_ASPACE) 819 cachep = gfs2_glock_aspace_cachep; 820 else 821 cachep = gfs2_glock_cachep; 822 gl = kmem_cache_alloc(cachep, GFP_NOFS); 823 if (!gl) 824 return -ENOMEM; 825 826 memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb)); 827 828 if (glops->go_flags & GLOF_LVB) { 829 gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS); 830 if (!gl->gl_lksb.sb_lvbptr) { 831 kmem_cache_free(cachep, gl); 832 return -ENOMEM; 833 } 834 } 835 836 atomic_inc(&sdp->sd_glock_disposal); 837 gl->gl_node.next = NULL; 838 gl->gl_flags = 0; 839 gl->gl_name = name; 840 gl->gl_lockref.count = 1; 841 gl->gl_state = LM_ST_UNLOCKED; 842 gl->gl_target = LM_ST_UNLOCKED; 843 gl->gl_demote_state = LM_ST_EXCLUSIVE; 844 gl->gl_ops = glops; 845 gl->gl_dstamp = 0; 846 preempt_disable(); 847 /* We use the global stats to estimate the initial per-glock stats */ 848 gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type]; 849 preempt_enable(); 850 gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0; 851 gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0; 852 gl->gl_tchange = jiffies; 853 gl->gl_object = NULL; 854 gl->gl_hold_time = GL_GLOCK_DFT_HOLD; 855 INIT_DELAYED_WORK(&gl->gl_work, glock_work_func); 856 INIT_WORK(&gl->gl_delete, delete_work_func); 857 858 mapping = gfs2_glock2aspace(gl); 859 if (mapping) { 860 mapping->a_ops = &gfs2_meta_aops; 861 mapping->host = s->s_bdev->bd_inode; 862 mapping->flags = 0; 863 mapping_set_gfp_mask(mapping, GFP_NOFS); 864 mapping->private_data = NULL; 865 mapping->writeback_index = 0; 866 } 867 868 tmp = find_insert_glock(&name, gl); 869 if (!tmp) { 870 *glp = gl; 871 goto out; 872 } 873 if (IS_ERR(tmp)) { 874 ret = PTR_ERR(tmp); 875 goto out_free; 876 } 877 *glp = tmp; 878 879 out_free: 880 kfree(gl->gl_lksb.sb_lvbptr); 881 kmem_cache_free(cachep, gl); 882 atomic_dec(&sdp->sd_glock_disposal); 883 884 out: 885 return ret; 886 } 887 888 /** 889 * gfs2_holder_init - initialize a struct gfs2_holder in the default way 890 * @gl: the glock 891 * @state: the state we're requesting 892 * @flags: the modifier flags 893 * @gh: the holder structure 894 * 895 */ 896 897 void gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags, 898 struct gfs2_holder *gh) 899 { 900 INIT_LIST_HEAD(&gh->gh_list); 901 gh->gh_gl = gl; 902 gh->gh_ip = _RET_IP_; 903 gh->gh_owner_pid = get_pid(task_pid(current)); 904 gh->gh_state = state; 905 gh->gh_flags = flags; 906 gh->gh_error = 0; 907 gh->gh_iflags = 0; 908 gfs2_glock_hold(gl); 909 } 910 911 /** 912 * gfs2_holder_reinit - reinitialize a struct gfs2_holder so we can requeue it 913 * @state: the state we're requesting 914 * @flags: the modifier flags 915 * @gh: the holder structure 916 * 917 * Don't mess with the glock. 918 * 919 */ 920 921 void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh) 922 { 923 gh->gh_state = state; 924 gh->gh_flags = flags; 925 gh->gh_iflags = 0; 926 gh->gh_ip = _RET_IP_; 927 put_pid(gh->gh_owner_pid); 928 gh->gh_owner_pid = get_pid(task_pid(current)); 929 } 930 931 /** 932 * gfs2_holder_uninit - uninitialize a holder structure (drop glock reference) 933 * @gh: the holder structure 934 * 935 */ 936 937 void gfs2_holder_uninit(struct gfs2_holder *gh) 938 { 939 put_pid(gh->gh_owner_pid); 940 gfs2_glock_put(gh->gh_gl); 941 gfs2_holder_mark_uninitialized(gh); 942 gh->gh_ip = 0; 943 } 944 945 static void gfs2_glock_update_hold_time(struct gfs2_glock *gl, 946 unsigned long start_time) 947 { 948 /* Have we waited longer that a second? */ 949 if (time_after(jiffies, start_time + HZ)) { 950 /* Lengthen the minimum hold time. */ 951 gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR, 952 GL_GLOCK_MAX_HOLD); 953 } 954 } 955 956 /** 957 * gfs2_glock_wait - wait on a glock acquisition 958 * @gh: the glock holder 959 * 960 * Returns: 0 on success 961 */ 962 963 int gfs2_glock_wait(struct gfs2_holder *gh) 964 { 965 unsigned long start_time = jiffies; 966 967 might_sleep(); 968 wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE); 969 gfs2_glock_update_hold_time(gh->gh_gl, start_time); 970 return gh->gh_error; 971 } 972 973 static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs) 974 { 975 int i; 976 977 for (i = 0; i < num_gh; i++) 978 if (test_bit(HIF_WAIT, &ghs[i].gh_iflags)) 979 return 1; 980 return 0; 981 } 982 983 /** 984 * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions 985 * @num_gh: the number of holders in the array 986 * @ghs: the glock holder array 987 * 988 * Returns: 0 on success, meaning all glocks have been granted and are held. 989 * -ESTALE if the request timed out, meaning all glocks were released, 990 * and the caller should retry the operation. 991 */ 992 993 int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs) 994 { 995 struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd; 996 int i, ret = 0, timeout = 0; 997 unsigned long start_time = jiffies; 998 bool keep_waiting; 999 1000 might_sleep(); 1001 /* 1002 * Total up the (minimum hold time * 2) of all glocks and use that to 1003 * determine the max amount of time we should wait. 1004 */ 1005 for (i = 0; i < num_gh; i++) 1006 timeout += ghs[i].gh_gl->gl_hold_time << 1; 1007 1008 wait_for_dlm: 1009 if (!wait_event_timeout(sdp->sd_async_glock_wait, 1010 !glocks_pending(num_gh, ghs), timeout)) 1011 ret = -ESTALE; /* request timed out. */ 1012 1013 /* 1014 * If dlm granted all our requests, we need to adjust the glock 1015 * minimum hold time values according to how long we waited. 1016 * 1017 * If our request timed out, we need to repeatedly release any held 1018 * glocks we acquired thus far to allow dlm to acquire the remaining 1019 * glocks without deadlocking. We cannot currently cancel outstanding 1020 * glock acquisitions. 1021 * 1022 * The HIF_WAIT bit tells us which requests still need a response from 1023 * dlm. 1024 * 1025 * If dlm sent us any errors, we return the first error we find. 1026 */ 1027 keep_waiting = false; 1028 for (i = 0; i < num_gh; i++) { 1029 /* Skip holders we have already dequeued below. */ 1030 if (!gfs2_holder_queued(&ghs[i])) 1031 continue; 1032 /* Skip holders with a pending DLM response. */ 1033 if (test_bit(HIF_WAIT, &ghs[i].gh_iflags)) { 1034 keep_waiting = true; 1035 continue; 1036 } 1037 1038 if (test_bit(HIF_HOLDER, &ghs[i].gh_iflags)) { 1039 if (ret == -ESTALE) 1040 gfs2_glock_dq(&ghs[i]); 1041 else 1042 gfs2_glock_update_hold_time(ghs[i].gh_gl, 1043 start_time); 1044 } 1045 if (!ret) 1046 ret = ghs[i].gh_error; 1047 } 1048 1049 if (keep_waiting) 1050 goto wait_for_dlm; 1051 1052 /* 1053 * At this point, we've either acquired all locks or released them all. 1054 */ 1055 return ret; 1056 } 1057 1058 /** 1059 * handle_callback - process a demote request 1060 * @gl: the glock 1061 * @state: the state the caller wants us to change to 1062 * 1063 * There are only two requests that we are going to see in actual 1064 * practise: LM_ST_SHARED and LM_ST_UNLOCKED 1065 */ 1066 1067 static void handle_callback(struct gfs2_glock *gl, unsigned int state, 1068 unsigned long delay, bool remote) 1069 { 1070 int bit = delay ? GLF_PENDING_DEMOTE : GLF_DEMOTE; 1071 1072 set_bit(bit, &gl->gl_flags); 1073 if (gl->gl_demote_state == LM_ST_EXCLUSIVE) { 1074 gl->gl_demote_state = state; 1075 gl->gl_demote_time = jiffies; 1076 } else if (gl->gl_demote_state != LM_ST_UNLOCKED && 1077 gl->gl_demote_state != state) { 1078 gl->gl_demote_state = LM_ST_UNLOCKED; 1079 } 1080 if (gl->gl_ops->go_callback) 1081 gl->gl_ops->go_callback(gl, remote); 1082 trace_gfs2_demote_rq(gl, remote); 1083 } 1084 1085 void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...) 1086 { 1087 struct va_format vaf; 1088 va_list args; 1089 1090 va_start(args, fmt); 1091 1092 if (seq) { 1093 seq_vprintf(seq, fmt, args); 1094 } else { 1095 vaf.fmt = fmt; 1096 vaf.va = &args; 1097 1098 pr_err("%pV", &vaf); 1099 } 1100 1101 va_end(args); 1102 } 1103 1104 /** 1105 * add_to_queue - Add a holder to the wait queue (but look for recursion) 1106 * @gh: the holder structure to add 1107 * 1108 * Eventually we should move the recursive locking trap to a 1109 * debugging option or something like that. This is the fast 1110 * path and needs to have the minimum number of distractions. 1111 * 1112 */ 1113 1114 static inline void add_to_queue(struct gfs2_holder *gh) 1115 __releases(&gl->gl_lockref.lock) 1116 __acquires(&gl->gl_lockref.lock) 1117 { 1118 struct gfs2_glock *gl = gh->gh_gl; 1119 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; 1120 struct list_head *insert_pt = NULL; 1121 struct gfs2_holder *gh2; 1122 int try_futile = 0; 1123 1124 GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL); 1125 if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags)) 1126 GLOCK_BUG_ON(gl, true); 1127 1128 if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) { 1129 if (test_bit(GLF_LOCK, &gl->gl_flags)) 1130 try_futile = !may_grant(gl, gh); 1131 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) 1132 goto fail; 1133 } 1134 1135 list_for_each_entry(gh2, &gl->gl_holders, gh_list) { 1136 if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid && 1137 (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK))) 1138 goto trap_recursive; 1139 if (try_futile && 1140 !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) { 1141 fail: 1142 gh->gh_error = GLR_TRYFAILED; 1143 gfs2_holder_wake(gh); 1144 return; 1145 } 1146 if (test_bit(HIF_HOLDER, &gh2->gh_iflags)) 1147 continue; 1148 if (unlikely((gh->gh_flags & LM_FLAG_PRIORITY) && !insert_pt)) 1149 insert_pt = &gh2->gh_list; 1150 } 1151 set_bit(GLF_QUEUED, &gl->gl_flags); 1152 trace_gfs2_glock_queue(gh, 1); 1153 gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT); 1154 gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT); 1155 if (likely(insert_pt == NULL)) { 1156 list_add_tail(&gh->gh_list, &gl->gl_holders); 1157 if (unlikely(gh->gh_flags & LM_FLAG_PRIORITY)) 1158 goto do_cancel; 1159 return; 1160 } 1161 list_add_tail(&gh->gh_list, insert_pt); 1162 do_cancel: 1163 gh = list_entry(gl->gl_holders.next, struct gfs2_holder, gh_list); 1164 if (!(gh->gh_flags & LM_FLAG_PRIORITY)) { 1165 spin_unlock(&gl->gl_lockref.lock); 1166 if (sdp->sd_lockstruct.ls_ops->lm_cancel) 1167 sdp->sd_lockstruct.ls_ops->lm_cancel(gl); 1168 spin_lock(&gl->gl_lockref.lock); 1169 } 1170 return; 1171 1172 trap_recursive: 1173 fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip); 1174 fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid)); 1175 fs_err(sdp, "lock type: %d req lock state : %d\n", 1176 gh2->gh_gl->gl_name.ln_type, gh2->gh_state); 1177 fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip); 1178 fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid)); 1179 fs_err(sdp, "lock type: %d req lock state : %d\n", 1180 gh->gh_gl->gl_name.ln_type, gh->gh_state); 1181 gfs2_dump_glock(NULL, gl, true); 1182 BUG(); 1183 } 1184 1185 /** 1186 * gfs2_glock_nq - enqueue a struct gfs2_holder onto a glock (acquire a glock) 1187 * @gh: the holder structure 1188 * 1189 * if (gh->gh_flags & GL_ASYNC), this never returns an error 1190 * 1191 * Returns: 0, GLR_TRYFAILED, or errno on failure 1192 */ 1193 1194 int gfs2_glock_nq(struct gfs2_holder *gh) 1195 { 1196 struct gfs2_glock *gl = gh->gh_gl; 1197 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; 1198 int error = 0; 1199 1200 if (unlikely(gfs2_withdrawn(sdp))) 1201 return -EIO; 1202 1203 if (test_bit(GLF_LRU, &gl->gl_flags)) 1204 gfs2_glock_remove_from_lru(gl); 1205 1206 spin_lock(&gl->gl_lockref.lock); 1207 add_to_queue(gh); 1208 if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) && 1209 test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) { 1210 set_bit(GLF_REPLY_PENDING, &gl->gl_flags); 1211 gl->gl_lockref.count++; 1212 __gfs2_glock_queue_work(gl, 0); 1213 } 1214 run_queue(gl, 1); 1215 spin_unlock(&gl->gl_lockref.lock); 1216 1217 if (!(gh->gh_flags & GL_ASYNC)) 1218 error = gfs2_glock_wait(gh); 1219 1220 return error; 1221 } 1222 1223 /** 1224 * gfs2_glock_poll - poll to see if an async request has been completed 1225 * @gh: the holder 1226 * 1227 * Returns: 1 if the request is ready to be gfs2_glock_wait()ed on 1228 */ 1229 1230 int gfs2_glock_poll(struct gfs2_holder *gh) 1231 { 1232 return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1; 1233 } 1234 1235 /** 1236 * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock) 1237 * @gh: the glock holder 1238 * 1239 */ 1240 1241 void gfs2_glock_dq(struct gfs2_holder *gh) 1242 { 1243 struct gfs2_glock *gl = gh->gh_gl; 1244 const struct gfs2_glock_operations *glops = gl->gl_ops; 1245 unsigned delay = 0; 1246 int fast_path = 0; 1247 1248 spin_lock(&gl->gl_lockref.lock); 1249 if (gh->gh_flags & GL_NOCACHE) 1250 handle_callback(gl, LM_ST_UNLOCKED, 0, false); 1251 1252 list_del_init(&gh->gh_list); 1253 clear_bit(HIF_HOLDER, &gh->gh_iflags); 1254 if (find_first_holder(gl) == NULL) { 1255 if (glops->go_unlock) { 1256 GLOCK_BUG_ON(gl, test_and_set_bit(GLF_LOCK, &gl->gl_flags)); 1257 spin_unlock(&gl->gl_lockref.lock); 1258 glops->go_unlock(gh); 1259 spin_lock(&gl->gl_lockref.lock); 1260 clear_bit(GLF_LOCK, &gl->gl_flags); 1261 } 1262 if (list_empty(&gl->gl_holders) && 1263 !test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) && 1264 !test_bit(GLF_DEMOTE, &gl->gl_flags)) 1265 fast_path = 1; 1266 } 1267 if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl)) 1268 gfs2_glock_add_to_lru(gl); 1269 1270 trace_gfs2_glock_queue(gh, 0); 1271 if (unlikely(!fast_path)) { 1272 gl->gl_lockref.count++; 1273 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) && 1274 !test_bit(GLF_DEMOTE, &gl->gl_flags) && 1275 gl->gl_name.ln_type == LM_TYPE_INODE) 1276 delay = gl->gl_hold_time; 1277 __gfs2_glock_queue_work(gl, delay); 1278 } 1279 spin_unlock(&gl->gl_lockref.lock); 1280 } 1281 1282 void gfs2_glock_dq_wait(struct gfs2_holder *gh) 1283 { 1284 struct gfs2_glock *gl = gh->gh_gl; 1285 gfs2_glock_dq(gh); 1286 might_sleep(); 1287 wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE); 1288 } 1289 1290 /** 1291 * gfs2_glock_dq_uninit - dequeue a holder from a glock and initialize it 1292 * @gh: the holder structure 1293 * 1294 */ 1295 1296 void gfs2_glock_dq_uninit(struct gfs2_holder *gh) 1297 { 1298 gfs2_glock_dq(gh); 1299 gfs2_holder_uninit(gh); 1300 } 1301 1302 /** 1303 * gfs2_glock_nq_num - acquire a glock based on lock number 1304 * @sdp: the filesystem 1305 * @number: the lock number 1306 * @glops: the glock operations for the type of glock 1307 * @state: the state to acquire the glock in 1308 * @flags: modifier flags for the acquisition 1309 * @gh: the struct gfs2_holder 1310 * 1311 * Returns: errno 1312 */ 1313 1314 int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number, 1315 const struct gfs2_glock_operations *glops, 1316 unsigned int state, u16 flags, struct gfs2_holder *gh) 1317 { 1318 struct gfs2_glock *gl; 1319 int error; 1320 1321 error = gfs2_glock_get(sdp, number, glops, CREATE, &gl); 1322 if (!error) { 1323 error = gfs2_glock_nq_init(gl, state, flags, gh); 1324 gfs2_glock_put(gl); 1325 } 1326 1327 return error; 1328 } 1329 1330 /** 1331 * glock_compare - Compare two struct gfs2_glock structures for sorting 1332 * @arg_a: the first structure 1333 * @arg_b: the second structure 1334 * 1335 */ 1336 1337 static int glock_compare(const void *arg_a, const void *arg_b) 1338 { 1339 const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a; 1340 const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b; 1341 const struct lm_lockname *a = &gh_a->gh_gl->gl_name; 1342 const struct lm_lockname *b = &gh_b->gh_gl->gl_name; 1343 1344 if (a->ln_number > b->ln_number) 1345 return 1; 1346 if (a->ln_number < b->ln_number) 1347 return -1; 1348 BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type); 1349 return 0; 1350 } 1351 1352 /** 1353 * nq_m_sync - synchonously acquire more than one glock in deadlock free order 1354 * @num_gh: the number of structures 1355 * @ghs: an array of struct gfs2_holder structures 1356 * 1357 * Returns: 0 on success (all glocks acquired), 1358 * errno on failure (no glocks acquired) 1359 */ 1360 1361 static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs, 1362 struct gfs2_holder **p) 1363 { 1364 unsigned int x; 1365 int error = 0; 1366 1367 for (x = 0; x < num_gh; x++) 1368 p[x] = &ghs[x]; 1369 1370 sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL); 1371 1372 for (x = 0; x < num_gh; x++) { 1373 p[x]->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC); 1374 1375 error = gfs2_glock_nq(p[x]); 1376 if (error) { 1377 while (x--) 1378 gfs2_glock_dq(p[x]); 1379 break; 1380 } 1381 } 1382 1383 return error; 1384 } 1385 1386 /** 1387 * gfs2_glock_nq_m - acquire multiple glocks 1388 * @num_gh: the number of structures 1389 * @ghs: an array of struct gfs2_holder structures 1390 * 1391 * 1392 * Returns: 0 on success (all glocks acquired), 1393 * errno on failure (no glocks acquired) 1394 */ 1395 1396 int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs) 1397 { 1398 struct gfs2_holder *tmp[4]; 1399 struct gfs2_holder **pph = tmp; 1400 int error = 0; 1401 1402 switch(num_gh) { 1403 case 0: 1404 return 0; 1405 case 1: 1406 ghs->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC); 1407 return gfs2_glock_nq(ghs); 1408 default: 1409 if (num_gh <= 4) 1410 break; 1411 pph = kmalloc_array(num_gh, sizeof(struct gfs2_holder *), 1412 GFP_NOFS); 1413 if (!pph) 1414 return -ENOMEM; 1415 } 1416 1417 error = nq_m_sync(num_gh, ghs, pph); 1418 1419 if (pph != tmp) 1420 kfree(pph); 1421 1422 return error; 1423 } 1424 1425 /** 1426 * gfs2_glock_dq_m - release multiple glocks 1427 * @num_gh: the number of structures 1428 * @ghs: an array of struct gfs2_holder structures 1429 * 1430 */ 1431 1432 void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs) 1433 { 1434 while (num_gh--) 1435 gfs2_glock_dq(&ghs[num_gh]); 1436 } 1437 1438 void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state) 1439 { 1440 unsigned long delay = 0; 1441 unsigned long holdtime; 1442 unsigned long now = jiffies; 1443 1444 gfs2_glock_hold(gl); 1445 holdtime = gl->gl_tchange + gl->gl_hold_time; 1446 if (test_bit(GLF_QUEUED, &gl->gl_flags) && 1447 gl->gl_name.ln_type == LM_TYPE_INODE) { 1448 if (time_before(now, holdtime)) 1449 delay = holdtime - now; 1450 if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags)) 1451 delay = gl->gl_hold_time; 1452 } 1453 1454 spin_lock(&gl->gl_lockref.lock); 1455 handle_callback(gl, state, delay, true); 1456 __gfs2_glock_queue_work(gl, delay); 1457 spin_unlock(&gl->gl_lockref.lock); 1458 } 1459 1460 /** 1461 * gfs2_should_freeze - Figure out if glock should be frozen 1462 * @gl: The glock in question 1463 * 1464 * Glocks are not frozen if (a) the result of the dlm operation is 1465 * an error, (b) the locking operation was an unlock operation or 1466 * (c) if there is a "noexp" flagged request anywhere in the queue 1467 * 1468 * Returns: 1 if freezing should occur, 0 otherwise 1469 */ 1470 1471 static int gfs2_should_freeze(const struct gfs2_glock *gl) 1472 { 1473 const struct gfs2_holder *gh; 1474 1475 if (gl->gl_reply & ~LM_OUT_ST_MASK) 1476 return 0; 1477 if (gl->gl_target == LM_ST_UNLOCKED) 1478 return 0; 1479 1480 list_for_each_entry(gh, &gl->gl_holders, gh_list) { 1481 if (test_bit(HIF_HOLDER, &gh->gh_iflags)) 1482 continue; 1483 if (LM_FLAG_NOEXP & gh->gh_flags) 1484 return 0; 1485 } 1486 1487 return 1; 1488 } 1489 1490 /** 1491 * gfs2_glock_complete - Callback used by locking 1492 * @gl: Pointer to the glock 1493 * @ret: The return value from the dlm 1494 * 1495 * The gl_reply field is under the gl_lockref.lock lock so that it is ok 1496 * to use a bitfield shared with other glock state fields. 1497 */ 1498 1499 void gfs2_glock_complete(struct gfs2_glock *gl, int ret) 1500 { 1501 struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct; 1502 1503 spin_lock(&gl->gl_lockref.lock); 1504 gl->gl_reply = ret; 1505 1506 if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) { 1507 if (gfs2_should_freeze(gl)) { 1508 set_bit(GLF_FROZEN, &gl->gl_flags); 1509 spin_unlock(&gl->gl_lockref.lock); 1510 return; 1511 } 1512 } 1513 1514 gl->gl_lockref.count++; 1515 set_bit(GLF_REPLY_PENDING, &gl->gl_flags); 1516 __gfs2_glock_queue_work(gl, 0); 1517 spin_unlock(&gl->gl_lockref.lock); 1518 } 1519 1520 static int glock_cmp(void *priv, struct list_head *a, struct list_head *b) 1521 { 1522 struct gfs2_glock *gla, *glb; 1523 1524 gla = list_entry(a, struct gfs2_glock, gl_lru); 1525 glb = list_entry(b, struct gfs2_glock, gl_lru); 1526 1527 if (gla->gl_name.ln_number > glb->gl_name.ln_number) 1528 return 1; 1529 if (gla->gl_name.ln_number < glb->gl_name.ln_number) 1530 return -1; 1531 1532 return 0; 1533 } 1534 1535 /** 1536 * gfs2_dispose_glock_lru - Demote a list of glocks 1537 * @list: The list to dispose of 1538 * 1539 * Disposing of glocks may involve disk accesses, so that here we sort 1540 * the glocks by number (i.e. disk location of the inodes) so that if 1541 * there are any such accesses, they'll be sent in order (mostly). 1542 * 1543 * Must be called under the lru_lock, but may drop and retake this 1544 * lock. While the lru_lock is dropped, entries may vanish from the 1545 * list, but no new entries will appear on the list (since it is 1546 * private) 1547 */ 1548 1549 static void gfs2_dispose_glock_lru(struct list_head *list) 1550 __releases(&lru_lock) 1551 __acquires(&lru_lock) 1552 { 1553 struct gfs2_glock *gl; 1554 1555 list_sort(NULL, list, glock_cmp); 1556 1557 while(!list_empty(list)) { 1558 gl = list_entry(list->next, struct gfs2_glock, gl_lru); 1559 list_del_init(&gl->gl_lru); 1560 if (!spin_trylock(&gl->gl_lockref.lock)) { 1561 add_back_to_lru: 1562 list_add(&gl->gl_lru, &lru_list); 1563 set_bit(GLF_LRU, &gl->gl_flags); 1564 atomic_inc(&lru_count); 1565 continue; 1566 } 1567 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) { 1568 spin_unlock(&gl->gl_lockref.lock); 1569 goto add_back_to_lru; 1570 } 1571 gl->gl_lockref.count++; 1572 if (demote_ok(gl)) 1573 handle_callback(gl, LM_ST_UNLOCKED, 0, false); 1574 WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags)); 1575 __gfs2_glock_queue_work(gl, 0); 1576 spin_unlock(&gl->gl_lockref.lock); 1577 cond_resched_lock(&lru_lock); 1578 } 1579 } 1580 1581 /** 1582 * gfs2_scan_glock_lru - Scan the LRU looking for locks to demote 1583 * @nr: The number of entries to scan 1584 * 1585 * This function selects the entries on the LRU which are able to 1586 * be demoted, and then kicks off the process by calling 1587 * gfs2_dispose_glock_lru() above. 1588 */ 1589 1590 static long gfs2_scan_glock_lru(int nr) 1591 { 1592 struct gfs2_glock *gl; 1593 LIST_HEAD(skipped); 1594 LIST_HEAD(dispose); 1595 long freed = 0; 1596 1597 spin_lock(&lru_lock); 1598 while ((nr-- >= 0) && !list_empty(&lru_list)) { 1599 gl = list_entry(lru_list.next, struct gfs2_glock, gl_lru); 1600 1601 /* Test for being demotable */ 1602 if (!test_bit(GLF_LOCK, &gl->gl_flags)) { 1603 list_move(&gl->gl_lru, &dispose); 1604 atomic_dec(&lru_count); 1605 clear_bit(GLF_LRU, &gl->gl_flags); 1606 freed++; 1607 continue; 1608 } 1609 1610 list_move(&gl->gl_lru, &skipped); 1611 } 1612 list_splice(&skipped, &lru_list); 1613 if (!list_empty(&dispose)) 1614 gfs2_dispose_glock_lru(&dispose); 1615 spin_unlock(&lru_lock); 1616 1617 return freed; 1618 } 1619 1620 static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink, 1621 struct shrink_control *sc) 1622 { 1623 if (!(sc->gfp_mask & __GFP_FS)) 1624 return SHRINK_STOP; 1625 return gfs2_scan_glock_lru(sc->nr_to_scan); 1626 } 1627 1628 static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink, 1629 struct shrink_control *sc) 1630 { 1631 return vfs_pressure_ratio(atomic_read(&lru_count)); 1632 } 1633 1634 static struct shrinker glock_shrinker = { 1635 .seeks = DEFAULT_SEEKS, 1636 .count_objects = gfs2_glock_shrink_count, 1637 .scan_objects = gfs2_glock_shrink_scan, 1638 }; 1639 1640 /** 1641 * examine_bucket - Call a function for glock in a hash bucket 1642 * @examiner: the function 1643 * @sdp: the filesystem 1644 * @bucket: the bucket 1645 * 1646 * Note that the function can be called multiple times on the same 1647 * object. So the user must ensure that the function can cope with 1648 * that. 1649 */ 1650 1651 static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp) 1652 { 1653 struct gfs2_glock *gl; 1654 struct rhashtable_iter iter; 1655 1656 rhashtable_walk_enter(&gl_hash_table, &iter); 1657 1658 do { 1659 rhashtable_walk_start(&iter); 1660 1661 while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) 1662 if (gl->gl_name.ln_sbd == sdp && 1663 lockref_get_not_dead(&gl->gl_lockref)) 1664 examiner(gl); 1665 1666 rhashtable_walk_stop(&iter); 1667 } while (cond_resched(), gl == ERR_PTR(-EAGAIN)); 1668 1669 rhashtable_walk_exit(&iter); 1670 } 1671 1672 /** 1673 * thaw_glock - thaw out a glock which has an unprocessed reply waiting 1674 * @gl: The glock to thaw 1675 * 1676 */ 1677 1678 static void thaw_glock(struct gfs2_glock *gl) 1679 { 1680 if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags)) { 1681 gfs2_glock_put(gl); 1682 return; 1683 } 1684 set_bit(GLF_REPLY_PENDING, &gl->gl_flags); 1685 gfs2_glock_queue_work(gl, 0); 1686 } 1687 1688 /** 1689 * clear_glock - look at a glock and see if we can free it from glock cache 1690 * @gl: the glock to look at 1691 * 1692 */ 1693 1694 static void clear_glock(struct gfs2_glock *gl) 1695 { 1696 gfs2_glock_remove_from_lru(gl); 1697 1698 spin_lock(&gl->gl_lockref.lock); 1699 if (gl->gl_state != LM_ST_UNLOCKED) 1700 handle_callback(gl, LM_ST_UNLOCKED, 0, false); 1701 __gfs2_glock_queue_work(gl, 0); 1702 spin_unlock(&gl->gl_lockref.lock); 1703 } 1704 1705 /** 1706 * gfs2_glock_thaw - Thaw any frozen glocks 1707 * @sdp: The super block 1708 * 1709 */ 1710 1711 void gfs2_glock_thaw(struct gfs2_sbd *sdp) 1712 { 1713 glock_hash_walk(thaw_glock, sdp); 1714 } 1715 1716 static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid) 1717 { 1718 spin_lock(&gl->gl_lockref.lock); 1719 gfs2_dump_glock(seq, gl, fsid); 1720 spin_unlock(&gl->gl_lockref.lock); 1721 } 1722 1723 static void dump_glock_func(struct gfs2_glock *gl) 1724 { 1725 dump_glock(NULL, gl, true); 1726 } 1727 1728 /** 1729 * gfs2_gl_hash_clear - Empty out the glock hash table 1730 * @sdp: the filesystem 1731 * @wait: wait until it's all gone 1732 * 1733 * Called when unmounting the filesystem. 1734 */ 1735 1736 void gfs2_gl_hash_clear(struct gfs2_sbd *sdp) 1737 { 1738 set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags); 1739 flush_workqueue(glock_workqueue); 1740 glock_hash_walk(clear_glock, sdp); 1741 flush_workqueue(glock_workqueue); 1742 wait_event_timeout(sdp->sd_glock_wait, 1743 atomic_read(&sdp->sd_glock_disposal) == 0, 1744 HZ * 600); 1745 glock_hash_walk(dump_glock_func, sdp); 1746 } 1747 1748 void gfs2_glock_finish_truncate(struct gfs2_inode *ip) 1749 { 1750 struct gfs2_glock *gl = ip->i_gl; 1751 int ret; 1752 1753 ret = gfs2_truncatei_resume(ip); 1754 gfs2_assert_withdraw(gl->gl_name.ln_sbd, ret == 0); 1755 1756 spin_lock(&gl->gl_lockref.lock); 1757 clear_bit(GLF_LOCK, &gl->gl_flags); 1758 run_queue(gl, 1); 1759 spin_unlock(&gl->gl_lockref.lock); 1760 } 1761 1762 static const char *state2str(unsigned state) 1763 { 1764 switch(state) { 1765 case LM_ST_UNLOCKED: 1766 return "UN"; 1767 case LM_ST_SHARED: 1768 return "SH"; 1769 case LM_ST_DEFERRED: 1770 return "DF"; 1771 case LM_ST_EXCLUSIVE: 1772 return "EX"; 1773 } 1774 return "??"; 1775 } 1776 1777 static const char *hflags2str(char *buf, u16 flags, unsigned long iflags) 1778 { 1779 char *p = buf; 1780 if (flags & LM_FLAG_TRY) 1781 *p++ = 't'; 1782 if (flags & LM_FLAG_TRY_1CB) 1783 *p++ = 'T'; 1784 if (flags & LM_FLAG_NOEXP) 1785 *p++ = 'e'; 1786 if (flags & LM_FLAG_ANY) 1787 *p++ = 'A'; 1788 if (flags & LM_FLAG_PRIORITY) 1789 *p++ = 'p'; 1790 if (flags & GL_ASYNC) 1791 *p++ = 'a'; 1792 if (flags & GL_EXACT) 1793 *p++ = 'E'; 1794 if (flags & GL_NOCACHE) 1795 *p++ = 'c'; 1796 if (test_bit(HIF_HOLDER, &iflags)) 1797 *p++ = 'H'; 1798 if (test_bit(HIF_WAIT, &iflags)) 1799 *p++ = 'W'; 1800 if (test_bit(HIF_FIRST, &iflags)) 1801 *p++ = 'F'; 1802 *p = 0; 1803 return buf; 1804 } 1805 1806 /** 1807 * dump_holder - print information about a glock holder 1808 * @seq: the seq_file struct 1809 * @gh: the glock holder 1810 * @fs_id_buf: pointer to file system id (if requested) 1811 * 1812 */ 1813 1814 static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh, 1815 const char *fs_id_buf) 1816 { 1817 struct task_struct *gh_owner = NULL; 1818 char flags_buf[32]; 1819 1820 rcu_read_lock(); 1821 if (gh->gh_owner_pid) 1822 gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID); 1823 gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n", 1824 fs_id_buf, state2str(gh->gh_state), 1825 hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags), 1826 gh->gh_error, 1827 gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1, 1828 gh_owner ? gh_owner->comm : "(ended)", 1829 (void *)gh->gh_ip); 1830 rcu_read_unlock(); 1831 } 1832 1833 static const char *gflags2str(char *buf, const struct gfs2_glock *gl) 1834 { 1835 const unsigned long *gflags = &gl->gl_flags; 1836 char *p = buf; 1837 1838 if (test_bit(GLF_LOCK, gflags)) 1839 *p++ = 'l'; 1840 if (test_bit(GLF_DEMOTE, gflags)) 1841 *p++ = 'D'; 1842 if (test_bit(GLF_PENDING_DEMOTE, gflags)) 1843 *p++ = 'd'; 1844 if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags)) 1845 *p++ = 'p'; 1846 if (test_bit(GLF_DIRTY, gflags)) 1847 *p++ = 'y'; 1848 if (test_bit(GLF_LFLUSH, gflags)) 1849 *p++ = 'f'; 1850 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, gflags)) 1851 *p++ = 'i'; 1852 if (test_bit(GLF_REPLY_PENDING, gflags)) 1853 *p++ = 'r'; 1854 if (test_bit(GLF_INITIAL, gflags)) 1855 *p++ = 'I'; 1856 if (test_bit(GLF_FROZEN, gflags)) 1857 *p++ = 'F'; 1858 if (test_bit(GLF_QUEUED, gflags)) 1859 *p++ = 'q'; 1860 if (test_bit(GLF_LRU, gflags)) 1861 *p++ = 'L'; 1862 if (gl->gl_object) 1863 *p++ = 'o'; 1864 if (test_bit(GLF_BLOCKING, gflags)) 1865 *p++ = 'b'; 1866 *p = 0; 1867 return buf; 1868 } 1869 1870 /** 1871 * gfs2_dump_glock - print information about a glock 1872 * @seq: The seq_file struct 1873 * @gl: the glock 1874 * @fsid: If true, also dump the file system id 1875 * 1876 * The file format is as follows: 1877 * One line per object, capital letters are used to indicate objects 1878 * G = glock, I = Inode, R = rgrp, H = holder. Glocks are not indented, 1879 * other objects are indented by a single space and follow the glock to 1880 * which they are related. Fields are indicated by lower case letters 1881 * followed by a colon and the field value, except for strings which are in 1882 * [] so that its possible to see if they are composed of spaces for 1883 * example. The field's are n = number (id of the object), f = flags, 1884 * t = type, s = state, r = refcount, e = error, p = pid. 1885 * 1886 */ 1887 1888 void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid) 1889 { 1890 const struct gfs2_glock_operations *glops = gl->gl_ops; 1891 unsigned long long dtime; 1892 const struct gfs2_holder *gh; 1893 char gflags_buf[32]; 1894 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; 1895 char fs_id_buf[sizeof(sdp->sd_fsname) + 7]; 1896 1897 memset(fs_id_buf, 0, sizeof(fs_id_buf)); 1898 if (fsid && sdp) /* safety precaution */ 1899 sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname); 1900 dtime = jiffies - gl->gl_demote_time; 1901 dtime *= 1000000/HZ; /* demote time in uSec */ 1902 if (!test_bit(GLF_DEMOTE, &gl->gl_flags)) 1903 dtime = 0; 1904 gfs2_print_dbg(seq, "%sG: s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d " 1905 "v:%d r:%d m:%ld\n", fs_id_buf, state2str(gl->gl_state), 1906 gl->gl_name.ln_type, 1907 (unsigned long long)gl->gl_name.ln_number, 1908 gflags2str(gflags_buf, gl), 1909 state2str(gl->gl_target), 1910 state2str(gl->gl_demote_state), dtime, 1911 atomic_read(&gl->gl_ail_count), 1912 atomic_read(&gl->gl_revokes), 1913 (int)gl->gl_lockref.count, gl->gl_hold_time); 1914 1915 list_for_each_entry(gh, &gl->gl_holders, gh_list) 1916 dump_holder(seq, gh, fs_id_buf); 1917 1918 if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump) 1919 glops->go_dump(seq, gl, fs_id_buf); 1920 } 1921 1922 static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr) 1923 { 1924 struct gfs2_glock *gl = iter_ptr; 1925 1926 seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n", 1927 gl->gl_name.ln_type, 1928 (unsigned long long)gl->gl_name.ln_number, 1929 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT], 1930 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR], 1931 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB], 1932 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB], 1933 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT], 1934 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR], 1935 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT], 1936 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]); 1937 return 0; 1938 } 1939 1940 static const char *gfs2_gltype[] = { 1941 "type", 1942 "reserved", 1943 "nondisk", 1944 "inode", 1945 "rgrp", 1946 "meta", 1947 "iopen", 1948 "flock", 1949 "plock", 1950 "quota", 1951 "journal", 1952 }; 1953 1954 static const char *gfs2_stype[] = { 1955 [GFS2_LKS_SRTT] = "srtt", 1956 [GFS2_LKS_SRTTVAR] = "srttvar", 1957 [GFS2_LKS_SRTTB] = "srttb", 1958 [GFS2_LKS_SRTTVARB] = "srttvarb", 1959 [GFS2_LKS_SIRT] = "sirt", 1960 [GFS2_LKS_SIRTVAR] = "sirtvar", 1961 [GFS2_LKS_DCOUNT] = "dlm", 1962 [GFS2_LKS_QCOUNT] = "queue", 1963 }; 1964 1965 #define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype)) 1966 1967 static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr) 1968 { 1969 struct gfs2_sbd *sdp = seq->private; 1970 loff_t pos = *(loff_t *)iter_ptr; 1971 unsigned index = pos >> 3; 1972 unsigned subindex = pos & 0x07; 1973 int i; 1974 1975 if (index == 0 && subindex != 0) 1976 return 0; 1977 1978 seq_printf(seq, "%-10s %8s:", gfs2_gltype[index], 1979 (index == 0) ? "cpu": gfs2_stype[subindex]); 1980 1981 for_each_possible_cpu(i) { 1982 const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i); 1983 1984 if (index == 0) 1985 seq_printf(seq, " %15u", i); 1986 else 1987 seq_printf(seq, " %15llu", (unsigned long long)lkstats-> 1988 lkstats[index - 1].stats[subindex]); 1989 } 1990 seq_putc(seq, '\n'); 1991 return 0; 1992 } 1993 1994 int __init gfs2_glock_init(void) 1995 { 1996 int i, ret; 1997 1998 ret = rhashtable_init(&gl_hash_table, &ht_parms); 1999 if (ret < 0) 2000 return ret; 2001 2002 glock_workqueue = alloc_workqueue("glock_workqueue", WQ_MEM_RECLAIM | 2003 WQ_HIGHPRI | WQ_FREEZABLE, 0); 2004 if (!glock_workqueue) { 2005 rhashtable_destroy(&gl_hash_table); 2006 return -ENOMEM; 2007 } 2008 gfs2_delete_workqueue = alloc_workqueue("delete_workqueue", 2009 WQ_MEM_RECLAIM | WQ_FREEZABLE, 2010 0); 2011 if (!gfs2_delete_workqueue) { 2012 destroy_workqueue(glock_workqueue); 2013 rhashtable_destroy(&gl_hash_table); 2014 return -ENOMEM; 2015 } 2016 2017 ret = register_shrinker(&glock_shrinker); 2018 if (ret) { 2019 destroy_workqueue(gfs2_delete_workqueue); 2020 destroy_workqueue(glock_workqueue); 2021 rhashtable_destroy(&gl_hash_table); 2022 return ret; 2023 } 2024 2025 for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++) 2026 init_waitqueue_head(glock_wait_table + i); 2027 2028 return 0; 2029 } 2030 2031 void gfs2_glock_exit(void) 2032 { 2033 unregister_shrinker(&glock_shrinker); 2034 rhashtable_destroy(&gl_hash_table); 2035 destroy_workqueue(glock_workqueue); 2036 destroy_workqueue(gfs2_delete_workqueue); 2037 } 2038 2039 static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n) 2040 { 2041 struct gfs2_glock *gl = gi->gl; 2042 2043 if (gl) { 2044 if (n == 0) 2045 return; 2046 if (!lockref_put_not_zero(&gl->gl_lockref)) 2047 gfs2_glock_queue_put(gl); 2048 } 2049 for (;;) { 2050 gl = rhashtable_walk_next(&gi->hti); 2051 if (IS_ERR_OR_NULL(gl)) { 2052 if (gl == ERR_PTR(-EAGAIN)) { 2053 n = 1; 2054 continue; 2055 } 2056 gl = NULL; 2057 break; 2058 } 2059 if (gl->gl_name.ln_sbd != gi->sdp) 2060 continue; 2061 if (n <= 1) { 2062 if (!lockref_get_not_dead(&gl->gl_lockref)) 2063 continue; 2064 break; 2065 } else { 2066 if (__lockref_is_dead(&gl->gl_lockref)) 2067 continue; 2068 n--; 2069 } 2070 } 2071 gi->gl = gl; 2072 } 2073 2074 static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos) 2075 __acquires(RCU) 2076 { 2077 struct gfs2_glock_iter *gi = seq->private; 2078 loff_t n; 2079 2080 /* 2081 * We can either stay where we are, skip to the next hash table 2082 * entry, or start from the beginning. 2083 */ 2084 if (*pos < gi->last_pos) { 2085 rhashtable_walk_exit(&gi->hti); 2086 rhashtable_walk_enter(&gl_hash_table, &gi->hti); 2087 n = *pos + 1; 2088 } else { 2089 n = *pos - gi->last_pos; 2090 } 2091 2092 rhashtable_walk_start(&gi->hti); 2093 2094 gfs2_glock_iter_next(gi, n); 2095 gi->last_pos = *pos; 2096 return gi->gl; 2097 } 2098 2099 static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr, 2100 loff_t *pos) 2101 { 2102 struct gfs2_glock_iter *gi = seq->private; 2103 2104 (*pos)++; 2105 gi->last_pos = *pos; 2106 gfs2_glock_iter_next(gi, 1); 2107 return gi->gl; 2108 } 2109 2110 static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr) 2111 __releases(RCU) 2112 { 2113 struct gfs2_glock_iter *gi = seq->private; 2114 2115 rhashtable_walk_stop(&gi->hti); 2116 } 2117 2118 static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr) 2119 { 2120 dump_glock(seq, iter_ptr, false); 2121 return 0; 2122 } 2123 2124 static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos) 2125 { 2126 preempt_disable(); 2127 if (*pos >= GFS2_NR_SBSTATS) 2128 return NULL; 2129 return pos; 2130 } 2131 2132 static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr, 2133 loff_t *pos) 2134 { 2135 (*pos)++; 2136 if (*pos >= GFS2_NR_SBSTATS) 2137 return NULL; 2138 return pos; 2139 } 2140 2141 static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr) 2142 { 2143 preempt_enable(); 2144 } 2145 2146 static const struct seq_operations gfs2_glock_seq_ops = { 2147 .start = gfs2_glock_seq_start, 2148 .next = gfs2_glock_seq_next, 2149 .stop = gfs2_glock_seq_stop, 2150 .show = gfs2_glock_seq_show, 2151 }; 2152 2153 static const struct seq_operations gfs2_glstats_seq_ops = { 2154 .start = gfs2_glock_seq_start, 2155 .next = gfs2_glock_seq_next, 2156 .stop = gfs2_glock_seq_stop, 2157 .show = gfs2_glstats_seq_show, 2158 }; 2159 2160 static const struct seq_operations gfs2_sbstats_seq_ops = { 2161 .start = gfs2_sbstats_seq_start, 2162 .next = gfs2_sbstats_seq_next, 2163 .stop = gfs2_sbstats_seq_stop, 2164 .show = gfs2_sbstats_seq_show, 2165 }; 2166 2167 #define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL) 2168 2169 static int __gfs2_glocks_open(struct inode *inode, struct file *file, 2170 const struct seq_operations *ops) 2171 { 2172 int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter)); 2173 if (ret == 0) { 2174 struct seq_file *seq = file->private_data; 2175 struct gfs2_glock_iter *gi = seq->private; 2176 2177 gi->sdp = inode->i_private; 2178 seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN); 2179 if (seq->buf) 2180 seq->size = GFS2_SEQ_GOODSIZE; 2181 /* 2182 * Initially, we are "before" the first hash table entry; the 2183 * first call to rhashtable_walk_next gets us the first entry. 2184 */ 2185 gi->last_pos = -1; 2186 gi->gl = NULL; 2187 rhashtable_walk_enter(&gl_hash_table, &gi->hti); 2188 } 2189 return ret; 2190 } 2191 2192 static int gfs2_glocks_open(struct inode *inode, struct file *file) 2193 { 2194 return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops); 2195 } 2196 2197 static int gfs2_glocks_release(struct inode *inode, struct file *file) 2198 { 2199 struct seq_file *seq = file->private_data; 2200 struct gfs2_glock_iter *gi = seq->private; 2201 2202 if (gi->gl) 2203 gfs2_glock_put(gi->gl); 2204 rhashtable_walk_exit(&gi->hti); 2205 return seq_release_private(inode, file); 2206 } 2207 2208 static int gfs2_glstats_open(struct inode *inode, struct file *file) 2209 { 2210 return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops); 2211 } 2212 2213 static int gfs2_sbstats_open(struct inode *inode, struct file *file) 2214 { 2215 int ret = seq_open(file, &gfs2_sbstats_seq_ops); 2216 if (ret == 0) { 2217 struct seq_file *seq = file->private_data; 2218 seq->private = inode->i_private; /* sdp */ 2219 } 2220 return ret; 2221 } 2222 2223 static const struct file_operations gfs2_glocks_fops = { 2224 .owner = THIS_MODULE, 2225 .open = gfs2_glocks_open, 2226 .read = seq_read, 2227 .llseek = seq_lseek, 2228 .release = gfs2_glocks_release, 2229 }; 2230 2231 static const struct file_operations gfs2_glstats_fops = { 2232 .owner = THIS_MODULE, 2233 .open = gfs2_glstats_open, 2234 .read = seq_read, 2235 .llseek = seq_lseek, 2236 .release = gfs2_glocks_release, 2237 }; 2238 2239 static const struct file_operations gfs2_sbstats_fops = { 2240 .owner = THIS_MODULE, 2241 .open = gfs2_sbstats_open, 2242 .read = seq_read, 2243 .llseek = seq_lseek, 2244 .release = seq_release, 2245 }; 2246 2247 void gfs2_create_debugfs_file(struct gfs2_sbd *sdp) 2248 { 2249 sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root); 2250 2251 debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, 2252 &gfs2_glocks_fops); 2253 2254 debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, 2255 &gfs2_glstats_fops); 2256 2257 debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, 2258 &gfs2_sbstats_fops); 2259 } 2260 2261 void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp) 2262 { 2263 debugfs_remove_recursive(sdp->debugfs_dir); 2264 sdp->debugfs_dir = NULL; 2265 } 2266 2267 void gfs2_register_debugfs(void) 2268 { 2269 gfs2_root = debugfs_create_dir("gfs2", NULL); 2270 } 2271 2272 void gfs2_unregister_debugfs(void) 2273 { 2274 debugfs_remove(gfs2_root); 2275 gfs2_root = NULL; 2276 } 2277