xref: /openbmc/linux/fs/dlm/debug_fs.c (revision b34e08d5)
1 /******************************************************************************
2 *******************************************************************************
3 **
4 **  Copyright (C) 2005-2009 Red Hat, Inc.  All rights reserved.
5 **
6 **  This copyrighted material is made available to anyone wishing to use,
7 **  modify, copy, or redistribute it subject to the terms and conditions
8 **  of the GNU General Public License v.2.
9 **
10 *******************************************************************************
11 ******************************************************************************/
12 
13 #include <linux/pagemap.h>
14 #include <linux/seq_file.h>
15 #include <linux/module.h>
16 #include <linux/ctype.h>
17 #include <linux/debugfs.h>
18 #include <linux/slab.h>
19 
20 #include "dlm_internal.h"
21 #include "lock.h"
22 
23 #define DLM_DEBUG_BUF_LEN 4096
24 static char debug_buf[DLM_DEBUG_BUF_LEN];
25 static struct mutex debug_buf_lock;
26 
27 static struct dentry *dlm_root;
28 
29 static char *print_lockmode(int mode)
30 {
31 	switch (mode) {
32 	case DLM_LOCK_IV:
33 		return "--";
34 	case DLM_LOCK_NL:
35 		return "NL";
36 	case DLM_LOCK_CR:
37 		return "CR";
38 	case DLM_LOCK_CW:
39 		return "CW";
40 	case DLM_LOCK_PR:
41 		return "PR";
42 	case DLM_LOCK_PW:
43 		return "PW";
44 	case DLM_LOCK_EX:
45 		return "EX";
46 	default:
47 		return "??";
48 	}
49 }
50 
51 static int print_format1_lock(struct seq_file *s, struct dlm_lkb *lkb,
52 			      struct dlm_rsb *res)
53 {
54 	seq_printf(s, "%08x %s", lkb->lkb_id, print_lockmode(lkb->lkb_grmode));
55 
56 	if (lkb->lkb_status == DLM_LKSTS_CONVERT ||
57 	    lkb->lkb_status == DLM_LKSTS_WAITING)
58 		seq_printf(s, " (%s)", print_lockmode(lkb->lkb_rqmode));
59 
60 	if (lkb->lkb_nodeid) {
61 		if (lkb->lkb_nodeid != res->res_nodeid)
62 			seq_printf(s, " Remote: %3d %08x", lkb->lkb_nodeid,
63 				   lkb->lkb_remid);
64 		else
65 			seq_printf(s, " Master:     %08x", lkb->lkb_remid);
66 	}
67 
68 	if (lkb->lkb_wait_type)
69 		seq_printf(s, " wait_type: %d", lkb->lkb_wait_type);
70 
71 	return seq_printf(s, "\n");
72 }
73 
74 static int print_format1(struct dlm_rsb *res, struct seq_file *s)
75 {
76 	struct dlm_lkb *lkb;
77 	int i, lvblen = res->res_ls->ls_lvblen, recover_list, root_list;
78 	int rv;
79 
80 	lock_rsb(res);
81 
82 	rv = seq_printf(s, "\nResource %p Name (len=%d) \"",
83 			res, res->res_length);
84 	if (rv)
85 		goto out;
86 
87 	for (i = 0; i < res->res_length; i++) {
88 		if (isprint(res->res_name[i]))
89 			seq_printf(s, "%c", res->res_name[i]);
90 		else
91 			seq_printf(s, "%c", '.');
92 	}
93 
94 	if (res->res_nodeid > 0)
95 		rv = seq_printf(s, "\"  \nLocal Copy, Master is node %d\n",
96 				res->res_nodeid);
97 	else if (res->res_nodeid == 0)
98 		rv = seq_printf(s, "\"  \nMaster Copy\n");
99 	else if (res->res_nodeid == -1)
100 		rv = seq_printf(s, "\"  \nLooking up master (lkid %x)\n",
101 			   	res->res_first_lkid);
102 	else
103 		rv = seq_printf(s, "\"  \nInvalid master %d\n",
104 				res->res_nodeid);
105 	if (rv)
106 		goto out;
107 
108 	/* Print the LVB: */
109 	if (res->res_lvbptr) {
110 		seq_printf(s, "LVB: ");
111 		for (i = 0; i < lvblen; i++) {
112 			if (i == lvblen / 2)
113 				seq_printf(s, "\n     ");
114 			seq_printf(s, "%02x ",
115 				   (unsigned char) res->res_lvbptr[i]);
116 		}
117 		if (rsb_flag(res, RSB_VALNOTVALID))
118 			seq_printf(s, " (INVALID)");
119 		rv = seq_printf(s, "\n");
120 		if (rv)
121 			goto out;
122 	}
123 
124 	root_list = !list_empty(&res->res_root_list);
125 	recover_list = !list_empty(&res->res_recover_list);
126 
127 	if (root_list || recover_list) {
128 		rv = seq_printf(s, "Recovery: root %d recover %d flags %lx "
129 				"count %d\n", root_list, recover_list,
130 			   	res->res_flags, res->res_recover_locks_count);
131 		if (rv)
132 			goto out;
133 	}
134 
135 	/* Print the locks attached to this resource */
136 	seq_printf(s, "Granted Queue\n");
137 	list_for_each_entry(lkb, &res->res_grantqueue, lkb_statequeue) {
138 		rv = print_format1_lock(s, lkb, res);
139 		if (rv)
140 			goto out;
141 	}
142 
143 	seq_printf(s, "Conversion Queue\n");
144 	list_for_each_entry(lkb, &res->res_convertqueue, lkb_statequeue) {
145 		rv = print_format1_lock(s, lkb, res);
146 		if (rv)
147 			goto out;
148 	}
149 
150 	seq_printf(s, "Waiting Queue\n");
151 	list_for_each_entry(lkb, &res->res_waitqueue, lkb_statequeue) {
152 		rv = print_format1_lock(s, lkb, res);
153 		if (rv)
154 			goto out;
155 	}
156 
157 	if (list_empty(&res->res_lookup))
158 		goto out;
159 
160 	seq_printf(s, "Lookup Queue\n");
161 	list_for_each_entry(lkb, &res->res_lookup, lkb_rsb_lookup) {
162 		rv = seq_printf(s, "%08x %s", lkb->lkb_id,
163 				print_lockmode(lkb->lkb_rqmode));
164 		if (lkb->lkb_wait_type)
165 			seq_printf(s, " wait_type: %d", lkb->lkb_wait_type);
166 		rv = seq_printf(s, "\n");
167 	}
168  out:
169 	unlock_rsb(res);
170 	return rv;
171 }
172 
173 static int print_format2_lock(struct seq_file *s, struct dlm_lkb *lkb,
174 			      struct dlm_rsb *r)
175 {
176 	u64 xid = 0;
177 	u64 us;
178 	int rv;
179 
180 	if (lkb->lkb_flags & DLM_IFL_USER) {
181 		if (lkb->lkb_ua)
182 			xid = lkb->lkb_ua->xid;
183 	}
184 
185 	/* microseconds since lkb was added to current queue */
186 	us = ktime_to_us(ktime_sub(ktime_get(), lkb->lkb_timestamp));
187 
188 	/* id nodeid remid pid xid exflags flags sts grmode rqmode time_us
189 	   r_nodeid r_len r_name */
190 
191 	rv = seq_printf(s, "%x %d %x %u %llu %x %x %d %d %d %llu %u %d \"%s\"\n",
192 			lkb->lkb_id,
193 			lkb->lkb_nodeid,
194 			lkb->lkb_remid,
195 			lkb->lkb_ownpid,
196 			(unsigned long long)xid,
197 			lkb->lkb_exflags,
198 			lkb->lkb_flags,
199 			lkb->lkb_status,
200 			lkb->lkb_grmode,
201 			lkb->lkb_rqmode,
202 			(unsigned long long)us,
203 			r->res_nodeid,
204 			r->res_length,
205 			r->res_name);
206 	return rv;
207 }
208 
209 static int print_format2(struct dlm_rsb *r, struct seq_file *s)
210 {
211 	struct dlm_lkb *lkb;
212 	int rv = 0;
213 
214 	lock_rsb(r);
215 
216 	list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) {
217 		rv = print_format2_lock(s, lkb, r);
218 		if (rv)
219 			goto out;
220 	}
221 
222 	list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) {
223 		rv = print_format2_lock(s, lkb, r);
224 		if (rv)
225 			goto out;
226 	}
227 
228 	list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) {
229 		rv = print_format2_lock(s, lkb, r);
230 		if (rv)
231 			goto out;
232 	}
233  out:
234 	unlock_rsb(r);
235 	return rv;
236 }
237 
238 static int print_format3_lock(struct seq_file *s, struct dlm_lkb *lkb,
239 			      int rsb_lookup)
240 {
241 	u64 xid = 0;
242 	int rv;
243 
244 	if (lkb->lkb_flags & DLM_IFL_USER) {
245 		if (lkb->lkb_ua)
246 			xid = lkb->lkb_ua->xid;
247 	}
248 
249 	rv = seq_printf(s, "lkb %x %d %x %u %llu %x %x %d %d %d %d %d %d %u %llu %llu\n",
250 			lkb->lkb_id,
251 			lkb->lkb_nodeid,
252 			lkb->lkb_remid,
253 			lkb->lkb_ownpid,
254 			(unsigned long long)xid,
255 			lkb->lkb_exflags,
256 			lkb->lkb_flags,
257 			lkb->lkb_status,
258 			lkb->lkb_grmode,
259 			lkb->lkb_rqmode,
260 			lkb->lkb_last_bast.mode,
261 			rsb_lookup,
262 			lkb->lkb_wait_type,
263 			lkb->lkb_lvbseq,
264 			(unsigned long long)ktime_to_ns(lkb->lkb_timestamp),
265 			(unsigned long long)ktime_to_ns(lkb->lkb_last_bast_time));
266 	return rv;
267 }
268 
269 static int print_format3(struct dlm_rsb *r, struct seq_file *s)
270 {
271 	struct dlm_lkb *lkb;
272 	int i, lvblen = r->res_ls->ls_lvblen;
273 	int print_name = 1;
274 	int rv;
275 
276 	lock_rsb(r);
277 
278 	rv = seq_printf(s, "rsb %p %d %x %lx %d %d %u %d ",
279 			r,
280 			r->res_nodeid,
281 			r->res_first_lkid,
282 			r->res_flags,
283 			!list_empty(&r->res_root_list),
284 			!list_empty(&r->res_recover_list),
285 			r->res_recover_locks_count,
286 			r->res_length);
287 	if (rv)
288 		goto out;
289 
290 	for (i = 0; i < r->res_length; i++) {
291 		if (!isascii(r->res_name[i]) || !isprint(r->res_name[i]))
292 			print_name = 0;
293 	}
294 
295 	seq_printf(s, "%s", print_name ? "str " : "hex");
296 
297 	for (i = 0; i < r->res_length; i++) {
298 		if (print_name)
299 			seq_printf(s, "%c", r->res_name[i]);
300 		else
301 			seq_printf(s, " %02x", (unsigned char)r->res_name[i]);
302 	}
303 	rv = seq_printf(s, "\n");
304 	if (rv)
305 		goto out;
306 
307 	if (!r->res_lvbptr)
308 		goto do_locks;
309 
310 	seq_printf(s, "lvb %u %d", r->res_lvbseq, lvblen);
311 
312 	for (i = 0; i < lvblen; i++)
313 		seq_printf(s, " %02x", (unsigned char)r->res_lvbptr[i]);
314 	rv = seq_printf(s, "\n");
315 	if (rv)
316 		goto out;
317 
318  do_locks:
319 	list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) {
320 		rv = print_format3_lock(s, lkb, 0);
321 		if (rv)
322 			goto out;
323 	}
324 
325 	list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) {
326 		rv = print_format3_lock(s, lkb, 0);
327 		if (rv)
328 			goto out;
329 	}
330 
331 	list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) {
332 		rv = print_format3_lock(s, lkb, 0);
333 		if (rv)
334 			goto out;
335 	}
336 
337 	list_for_each_entry(lkb, &r->res_lookup, lkb_rsb_lookup) {
338 		rv = print_format3_lock(s, lkb, 1);
339 		if (rv)
340 			goto out;
341 	}
342  out:
343 	unlock_rsb(r);
344 	return rv;
345 }
346 
347 static int print_format4(struct dlm_rsb *r, struct seq_file *s)
348 {
349 	int our_nodeid = dlm_our_nodeid();
350 	int print_name = 1;
351 	int i, rv;
352 
353 	lock_rsb(r);
354 
355 	rv = seq_printf(s, "rsb %p %d %d %d %d %lu %lx %d ",
356 			r,
357 			r->res_nodeid,
358 			r->res_master_nodeid,
359 			r->res_dir_nodeid,
360 			our_nodeid,
361 			r->res_toss_time,
362 			r->res_flags,
363 			r->res_length);
364 	if (rv)
365 		goto out;
366 
367 	for (i = 0; i < r->res_length; i++) {
368 		if (!isascii(r->res_name[i]) || !isprint(r->res_name[i]))
369 			print_name = 0;
370 	}
371 
372 	seq_printf(s, "%s", print_name ? "str " : "hex");
373 
374 	for (i = 0; i < r->res_length; i++) {
375 		if (print_name)
376 			seq_printf(s, "%c", r->res_name[i]);
377 		else
378 			seq_printf(s, " %02x", (unsigned char)r->res_name[i]);
379 	}
380 	rv = seq_printf(s, "\n");
381  out:
382 	unlock_rsb(r);
383 	return rv;
384 }
385 
386 struct rsbtbl_iter {
387 	struct dlm_rsb *rsb;
388 	unsigned bucket;
389 	int format;
390 	int header;
391 };
392 
393 /* seq_printf returns -1 if the buffer is full, and 0 otherwise.
394    If the buffer is full, seq_printf can be called again, but it
395    does nothing and just returns -1.  So, the these printing routines
396    periodically check the return value to avoid wasting too much time
397    trying to print to a full buffer. */
398 
399 static int table_seq_show(struct seq_file *seq, void *iter_ptr)
400 {
401 	struct rsbtbl_iter *ri = iter_ptr;
402 	int rv = 0;
403 
404 	switch (ri->format) {
405 	case 1:
406 		rv = print_format1(ri->rsb, seq);
407 		break;
408 	case 2:
409 		if (ri->header) {
410 			seq_printf(seq, "id nodeid remid pid xid exflags "
411 					"flags sts grmode rqmode time_ms "
412 					"r_nodeid r_len r_name\n");
413 			ri->header = 0;
414 		}
415 		rv = print_format2(ri->rsb, seq);
416 		break;
417 	case 3:
418 		if (ri->header) {
419 			seq_printf(seq, "version rsb 1.1 lvb 1.1 lkb 1.1\n");
420 			ri->header = 0;
421 		}
422 		rv = print_format3(ri->rsb, seq);
423 		break;
424 	case 4:
425 		if (ri->header) {
426 			seq_printf(seq, "version 4 rsb 2\n");
427 			ri->header = 0;
428 		}
429 		rv = print_format4(ri->rsb, seq);
430 		break;
431 	}
432 
433 	return rv;
434 }
435 
436 static const struct seq_operations format1_seq_ops;
437 static const struct seq_operations format2_seq_ops;
438 static const struct seq_operations format3_seq_ops;
439 static const struct seq_operations format4_seq_ops;
440 
441 static void *table_seq_start(struct seq_file *seq, loff_t *pos)
442 {
443 	struct rb_root *tree;
444 	struct rb_node *node;
445 	struct dlm_ls *ls = seq->private;
446 	struct rsbtbl_iter *ri;
447 	struct dlm_rsb *r;
448 	loff_t n = *pos;
449 	unsigned bucket, entry;
450 	int toss = (seq->op == &format4_seq_ops);
451 
452 	bucket = n >> 32;
453 	entry = n & ((1LL << 32) - 1);
454 
455 	if (bucket >= ls->ls_rsbtbl_size)
456 		return NULL;
457 
458 	ri = kzalloc(sizeof(struct rsbtbl_iter), GFP_NOFS);
459 	if (!ri)
460 		return NULL;
461 	if (n == 0)
462 		ri->header = 1;
463 	if (seq->op == &format1_seq_ops)
464 		ri->format = 1;
465 	if (seq->op == &format2_seq_ops)
466 		ri->format = 2;
467 	if (seq->op == &format3_seq_ops)
468 		ri->format = 3;
469 	if (seq->op == &format4_seq_ops)
470 		ri->format = 4;
471 
472 	tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep;
473 
474 	spin_lock(&ls->ls_rsbtbl[bucket].lock);
475 	if (!RB_EMPTY_ROOT(tree)) {
476 		for (node = rb_first(tree); node; node = rb_next(node)) {
477 			r = rb_entry(node, struct dlm_rsb, res_hashnode);
478 			if (!entry--) {
479 				dlm_hold_rsb(r);
480 				ri->rsb = r;
481 				ri->bucket = bucket;
482 				spin_unlock(&ls->ls_rsbtbl[bucket].lock);
483 				return ri;
484 			}
485 		}
486 	}
487 	spin_unlock(&ls->ls_rsbtbl[bucket].lock);
488 
489 	/*
490 	 * move to the first rsb in the next non-empty bucket
491 	 */
492 
493 	/* zero the entry */
494 	n &= ~((1LL << 32) - 1);
495 
496 	while (1) {
497 		bucket++;
498 		n += 1LL << 32;
499 
500 		if (bucket >= ls->ls_rsbtbl_size) {
501 			kfree(ri);
502 			return NULL;
503 		}
504 		tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep;
505 
506 		spin_lock(&ls->ls_rsbtbl[bucket].lock);
507 		if (!RB_EMPTY_ROOT(tree)) {
508 			node = rb_first(tree);
509 			r = rb_entry(node, struct dlm_rsb, res_hashnode);
510 			dlm_hold_rsb(r);
511 			ri->rsb = r;
512 			ri->bucket = bucket;
513 			spin_unlock(&ls->ls_rsbtbl[bucket].lock);
514 			*pos = n;
515 			return ri;
516 		}
517 		spin_unlock(&ls->ls_rsbtbl[bucket].lock);
518 	}
519 }
520 
521 static void *table_seq_next(struct seq_file *seq, void *iter_ptr, loff_t *pos)
522 {
523 	struct dlm_ls *ls = seq->private;
524 	struct rsbtbl_iter *ri = iter_ptr;
525 	struct rb_root *tree;
526 	struct rb_node *next;
527 	struct dlm_rsb *r, *rp;
528 	loff_t n = *pos;
529 	unsigned bucket;
530 	int toss = (seq->op == &format4_seq_ops);
531 
532 	bucket = n >> 32;
533 
534 	/*
535 	 * move to the next rsb in the same bucket
536 	 */
537 
538 	spin_lock(&ls->ls_rsbtbl[bucket].lock);
539 	rp = ri->rsb;
540 	next = rb_next(&rp->res_hashnode);
541 
542 	if (next) {
543 		r = rb_entry(next, struct dlm_rsb, res_hashnode);
544 		dlm_hold_rsb(r);
545 		ri->rsb = r;
546 		spin_unlock(&ls->ls_rsbtbl[bucket].lock);
547 		dlm_put_rsb(rp);
548 		++*pos;
549 		return ri;
550 	}
551 	spin_unlock(&ls->ls_rsbtbl[bucket].lock);
552 	dlm_put_rsb(rp);
553 
554 	/*
555 	 * move to the first rsb in the next non-empty bucket
556 	 */
557 
558 	/* zero the entry */
559 	n &= ~((1LL << 32) - 1);
560 
561 	while (1) {
562 		bucket++;
563 		n += 1LL << 32;
564 
565 		if (bucket >= ls->ls_rsbtbl_size) {
566 			kfree(ri);
567 			return NULL;
568 		}
569 		tree = toss ? &ls->ls_rsbtbl[bucket].toss : &ls->ls_rsbtbl[bucket].keep;
570 
571 		spin_lock(&ls->ls_rsbtbl[bucket].lock);
572 		if (!RB_EMPTY_ROOT(tree)) {
573 			next = rb_first(tree);
574 			r = rb_entry(next, struct dlm_rsb, res_hashnode);
575 			dlm_hold_rsb(r);
576 			ri->rsb = r;
577 			ri->bucket = bucket;
578 			spin_unlock(&ls->ls_rsbtbl[bucket].lock);
579 			*pos = n;
580 			return ri;
581 		}
582 		spin_unlock(&ls->ls_rsbtbl[bucket].lock);
583 	}
584 }
585 
586 static void table_seq_stop(struct seq_file *seq, void *iter_ptr)
587 {
588 	struct rsbtbl_iter *ri = iter_ptr;
589 
590 	if (ri) {
591 		dlm_put_rsb(ri->rsb);
592 		kfree(ri);
593 	}
594 }
595 
596 static const struct seq_operations format1_seq_ops = {
597 	.start = table_seq_start,
598 	.next  = table_seq_next,
599 	.stop  = table_seq_stop,
600 	.show  = table_seq_show,
601 };
602 
603 static const struct seq_operations format2_seq_ops = {
604 	.start = table_seq_start,
605 	.next  = table_seq_next,
606 	.stop  = table_seq_stop,
607 	.show  = table_seq_show,
608 };
609 
610 static const struct seq_operations format3_seq_ops = {
611 	.start = table_seq_start,
612 	.next  = table_seq_next,
613 	.stop  = table_seq_stop,
614 	.show  = table_seq_show,
615 };
616 
617 static const struct seq_operations format4_seq_ops = {
618 	.start = table_seq_start,
619 	.next  = table_seq_next,
620 	.stop  = table_seq_stop,
621 	.show  = table_seq_show,
622 };
623 
624 static const struct file_operations format1_fops;
625 static const struct file_operations format2_fops;
626 static const struct file_operations format3_fops;
627 static const struct file_operations format4_fops;
628 
629 static int table_open(struct inode *inode, struct file *file)
630 {
631 	struct seq_file *seq;
632 	int ret = -1;
633 
634 	if (file->f_op == &format1_fops)
635 		ret = seq_open(file, &format1_seq_ops);
636 	else if (file->f_op == &format2_fops)
637 		ret = seq_open(file, &format2_seq_ops);
638 	else if (file->f_op == &format3_fops)
639 		ret = seq_open(file, &format3_seq_ops);
640 	else if (file->f_op == &format4_fops)
641 		ret = seq_open(file, &format4_seq_ops);
642 
643 	if (ret)
644 		return ret;
645 
646 	seq = file->private_data;
647 	seq->private = inode->i_private; /* the dlm_ls */
648 	return 0;
649 }
650 
651 static const struct file_operations format1_fops = {
652 	.owner   = THIS_MODULE,
653 	.open    = table_open,
654 	.read    = seq_read,
655 	.llseek  = seq_lseek,
656 	.release = seq_release
657 };
658 
659 static const struct file_operations format2_fops = {
660 	.owner   = THIS_MODULE,
661 	.open    = table_open,
662 	.read    = seq_read,
663 	.llseek  = seq_lseek,
664 	.release = seq_release
665 };
666 
667 static const struct file_operations format3_fops = {
668 	.owner   = THIS_MODULE,
669 	.open    = table_open,
670 	.read    = seq_read,
671 	.llseek  = seq_lseek,
672 	.release = seq_release
673 };
674 
675 static const struct file_operations format4_fops = {
676 	.owner   = THIS_MODULE,
677 	.open    = table_open,
678 	.read    = seq_read,
679 	.llseek  = seq_lseek,
680 	.release = seq_release
681 };
682 
683 /*
684  * dump lkb's on the ls_waiters list
685  */
686 static ssize_t waiters_read(struct file *file, char __user *userbuf,
687 			    size_t count, loff_t *ppos)
688 {
689 	struct dlm_ls *ls = file->private_data;
690 	struct dlm_lkb *lkb;
691 	size_t len = DLM_DEBUG_BUF_LEN, pos = 0, ret, rv;
692 
693 	mutex_lock(&debug_buf_lock);
694 	mutex_lock(&ls->ls_waiters_mutex);
695 	memset(debug_buf, 0, sizeof(debug_buf));
696 
697 	list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) {
698 		ret = snprintf(debug_buf + pos, len - pos, "%x %d %d %s\n",
699 			       lkb->lkb_id, lkb->lkb_wait_type,
700 			       lkb->lkb_nodeid, lkb->lkb_resource->res_name);
701 		if (ret >= len - pos)
702 			break;
703 		pos += ret;
704 	}
705 	mutex_unlock(&ls->ls_waiters_mutex);
706 
707 	rv = simple_read_from_buffer(userbuf, count, ppos, debug_buf, pos);
708 	mutex_unlock(&debug_buf_lock);
709 	return rv;
710 }
711 
712 static const struct file_operations waiters_fops = {
713 	.owner   = THIS_MODULE,
714 	.open    = simple_open,
715 	.read    = waiters_read,
716 	.llseek  = default_llseek,
717 };
718 
719 void dlm_delete_debug_file(struct dlm_ls *ls)
720 {
721 	if (ls->ls_debug_rsb_dentry)
722 		debugfs_remove(ls->ls_debug_rsb_dentry);
723 	if (ls->ls_debug_waiters_dentry)
724 		debugfs_remove(ls->ls_debug_waiters_dentry);
725 	if (ls->ls_debug_locks_dentry)
726 		debugfs_remove(ls->ls_debug_locks_dentry);
727 	if (ls->ls_debug_all_dentry)
728 		debugfs_remove(ls->ls_debug_all_dentry);
729 	if (ls->ls_debug_toss_dentry)
730 		debugfs_remove(ls->ls_debug_toss_dentry);
731 }
732 
733 int dlm_create_debug_file(struct dlm_ls *ls)
734 {
735 	char name[DLM_LOCKSPACE_LEN+8];
736 
737 	/* format 1 */
738 
739 	ls->ls_debug_rsb_dentry = debugfs_create_file(ls->ls_name,
740 						      S_IFREG | S_IRUGO,
741 						      dlm_root,
742 						      ls,
743 						      &format1_fops);
744 	if (!ls->ls_debug_rsb_dentry)
745 		goto fail;
746 
747 	/* format 2 */
748 
749 	memset(name, 0, sizeof(name));
750 	snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_locks", ls->ls_name);
751 
752 	ls->ls_debug_locks_dentry = debugfs_create_file(name,
753 							S_IFREG | S_IRUGO,
754 							dlm_root,
755 							ls,
756 							&format2_fops);
757 	if (!ls->ls_debug_locks_dentry)
758 		goto fail;
759 
760 	/* format 3 */
761 
762 	memset(name, 0, sizeof(name));
763 	snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_all", ls->ls_name);
764 
765 	ls->ls_debug_all_dentry = debugfs_create_file(name,
766 						      S_IFREG | S_IRUGO,
767 						      dlm_root,
768 						      ls,
769 						      &format3_fops);
770 	if (!ls->ls_debug_all_dentry)
771 		goto fail;
772 
773 	/* format 4 */
774 
775 	memset(name, 0, sizeof(name));
776 	snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_toss", ls->ls_name);
777 
778 	ls->ls_debug_toss_dentry = debugfs_create_file(name,
779 						       S_IFREG | S_IRUGO,
780 						       dlm_root,
781 						       ls,
782 						       &format4_fops);
783 	if (!ls->ls_debug_toss_dentry)
784 		goto fail;
785 
786 	memset(name, 0, sizeof(name));
787 	snprintf(name, DLM_LOCKSPACE_LEN+8, "%s_waiters", ls->ls_name);
788 
789 	ls->ls_debug_waiters_dentry = debugfs_create_file(name,
790 							  S_IFREG | S_IRUGO,
791 							  dlm_root,
792 							  ls,
793 							  &waiters_fops);
794 	if (!ls->ls_debug_waiters_dentry)
795 		goto fail;
796 
797 	return 0;
798 
799  fail:
800 	dlm_delete_debug_file(ls);
801 	return -ENOMEM;
802 }
803 
804 int __init dlm_register_debugfs(void)
805 {
806 	mutex_init(&debug_buf_lock);
807 	dlm_root = debugfs_create_dir("dlm", NULL);
808 	return dlm_root ? 0 : -ENOMEM;
809 }
810 
811 void dlm_unregister_debugfs(void)
812 {
813 	debugfs_remove(dlm_root);
814 }
815 
816