1 /* 2 * Public API and common code for kernel->userspace relay file support. 3 * 4 * See Documentation/filesystems/relayfs.txt for an overview of relayfs. 5 * 6 * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp 7 * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com) 8 * 9 * Moved to kernel/relay.c by Paul Mundt, 2006. 10 * November 2006 - CPU hotplug support by Mathieu Desnoyers 11 * (mathieu.desnoyers@polymtl.ca) 12 * 13 * This file is released under the GPL. 14 */ 15 #include <linux/errno.h> 16 #include <linux/stddef.h> 17 #include <linux/slab.h> 18 #include <linux/module.h> 19 #include <linux/string.h> 20 #include <linux/relay.h> 21 #include <linux/vmalloc.h> 22 #include <linux/mm.h> 23 #include <linux/cpu.h> 24 25 /* list of open channels, for cpu hotplug */ 26 static DEFINE_MUTEX(relay_channels_mutex); 27 static LIST_HEAD(relay_channels); 28 29 /* 30 * close() vm_op implementation for relay file mapping. 31 */ 32 static void relay_file_mmap_close(struct vm_area_struct *vma) 33 { 34 struct rchan_buf *buf = vma->vm_private_data; 35 buf->chan->cb->buf_unmapped(buf, vma->vm_file); 36 } 37 38 /* 39 * nopage() vm_op implementation for relay file mapping. 40 */ 41 static struct page *relay_buf_nopage(struct vm_area_struct *vma, 42 unsigned long address, 43 int *type) 44 { 45 struct page *page; 46 struct rchan_buf *buf = vma->vm_private_data; 47 unsigned long offset = address - vma->vm_start; 48 49 if (address > vma->vm_end) 50 return NOPAGE_SIGBUS; /* Disallow mremap */ 51 if (!buf) 52 return NOPAGE_OOM; 53 54 page = vmalloc_to_page(buf->start + offset); 55 if (!page) 56 return NOPAGE_OOM; 57 get_page(page); 58 59 if (type) 60 *type = VM_FAULT_MINOR; 61 62 return page; 63 } 64 65 /* 66 * vm_ops for relay file mappings. 67 */ 68 static struct vm_operations_struct relay_file_mmap_ops = { 69 .nopage = relay_buf_nopage, 70 .close = relay_file_mmap_close, 71 }; 72 73 /** 74 * relay_mmap_buf: - mmap channel buffer to process address space 75 * @buf: relay channel buffer 76 * @vma: vm_area_struct describing memory to be mapped 77 * 78 * Returns 0 if ok, negative on error 79 * 80 * Caller should already have grabbed mmap_sem. 81 */ 82 int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma) 83 { 84 unsigned long length = vma->vm_end - vma->vm_start; 85 struct file *filp = vma->vm_file; 86 87 if (!buf) 88 return -EBADF; 89 90 if (length != (unsigned long)buf->chan->alloc_size) 91 return -EINVAL; 92 93 vma->vm_ops = &relay_file_mmap_ops; 94 vma->vm_private_data = buf; 95 buf->chan->cb->buf_mapped(buf, filp); 96 97 return 0; 98 } 99 100 /** 101 * relay_alloc_buf - allocate a channel buffer 102 * @buf: the buffer struct 103 * @size: total size of the buffer 104 * 105 * Returns a pointer to the resulting buffer, %NULL if unsuccessful. The 106 * passed in size will get page aligned, if it isn't already. 107 */ 108 static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size) 109 { 110 void *mem; 111 unsigned int i, j, n_pages; 112 113 *size = PAGE_ALIGN(*size); 114 n_pages = *size >> PAGE_SHIFT; 115 116 buf->page_array = kcalloc(n_pages, sizeof(struct page *), GFP_KERNEL); 117 if (!buf->page_array) 118 return NULL; 119 120 for (i = 0; i < n_pages; i++) { 121 buf->page_array[i] = alloc_page(GFP_KERNEL); 122 if (unlikely(!buf->page_array[i])) 123 goto depopulate; 124 } 125 mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL); 126 if (!mem) 127 goto depopulate; 128 129 memset(mem, 0, *size); 130 buf->page_count = n_pages; 131 return mem; 132 133 depopulate: 134 for (j = 0; j < i; j++) 135 __free_page(buf->page_array[j]); 136 kfree(buf->page_array); 137 return NULL; 138 } 139 140 /** 141 * relay_create_buf - allocate and initialize a channel buffer 142 * @chan: the relay channel 143 * 144 * Returns channel buffer if successful, %NULL otherwise. 145 */ 146 struct rchan_buf *relay_create_buf(struct rchan *chan) 147 { 148 struct rchan_buf *buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL); 149 if (!buf) 150 return NULL; 151 152 buf->padding = kmalloc(chan->n_subbufs * sizeof(size_t *), GFP_KERNEL); 153 if (!buf->padding) 154 goto free_buf; 155 156 buf->start = relay_alloc_buf(buf, &chan->alloc_size); 157 if (!buf->start) 158 goto free_buf; 159 160 buf->chan = chan; 161 kref_get(&buf->chan->kref); 162 return buf; 163 164 free_buf: 165 kfree(buf->padding); 166 kfree(buf); 167 return NULL; 168 } 169 170 /** 171 * relay_destroy_channel - free the channel struct 172 * @kref: target kernel reference that contains the relay channel 173 * 174 * Should only be called from kref_put(). 175 */ 176 void relay_destroy_channel(struct kref *kref) 177 { 178 struct rchan *chan = container_of(kref, struct rchan, kref); 179 kfree(chan); 180 } 181 182 /** 183 * relay_destroy_buf - destroy an rchan_buf struct and associated buffer 184 * @buf: the buffer struct 185 */ 186 void relay_destroy_buf(struct rchan_buf *buf) 187 { 188 struct rchan *chan = buf->chan; 189 unsigned int i; 190 191 if (likely(buf->start)) { 192 vunmap(buf->start); 193 for (i = 0; i < buf->page_count; i++) 194 __free_page(buf->page_array[i]); 195 kfree(buf->page_array); 196 } 197 chan->buf[buf->cpu] = NULL; 198 kfree(buf->padding); 199 kfree(buf); 200 kref_put(&chan->kref, relay_destroy_channel); 201 } 202 203 /** 204 * relay_remove_buf - remove a channel buffer 205 * @kref: target kernel reference that contains the relay buffer 206 * 207 * Removes the file from the fileystem, which also frees the 208 * rchan_buf_struct and the channel buffer. Should only be called from 209 * kref_put(). 210 */ 211 void relay_remove_buf(struct kref *kref) 212 { 213 struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref); 214 buf->chan->cb->remove_buf_file(buf->dentry); 215 relay_destroy_buf(buf); 216 } 217 218 /** 219 * relay_buf_empty - boolean, is the channel buffer empty? 220 * @buf: channel buffer 221 * 222 * Returns 1 if the buffer is empty, 0 otherwise. 223 */ 224 int relay_buf_empty(struct rchan_buf *buf) 225 { 226 return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1; 227 } 228 EXPORT_SYMBOL_GPL(relay_buf_empty); 229 230 /** 231 * relay_buf_full - boolean, is the channel buffer full? 232 * @buf: channel buffer 233 * 234 * Returns 1 if the buffer is full, 0 otherwise. 235 */ 236 int relay_buf_full(struct rchan_buf *buf) 237 { 238 size_t ready = buf->subbufs_produced - buf->subbufs_consumed; 239 return (ready >= buf->chan->n_subbufs) ? 1 : 0; 240 } 241 EXPORT_SYMBOL_GPL(relay_buf_full); 242 243 /* 244 * High-level relay kernel API and associated functions. 245 */ 246 247 /* 248 * rchan_callback implementations defining default channel behavior. Used 249 * in place of corresponding NULL values in client callback struct. 250 */ 251 252 /* 253 * subbuf_start() default callback. Does nothing. 254 */ 255 static int subbuf_start_default_callback (struct rchan_buf *buf, 256 void *subbuf, 257 void *prev_subbuf, 258 size_t prev_padding) 259 { 260 if (relay_buf_full(buf)) 261 return 0; 262 263 return 1; 264 } 265 266 /* 267 * buf_mapped() default callback. Does nothing. 268 */ 269 static void buf_mapped_default_callback(struct rchan_buf *buf, 270 struct file *filp) 271 { 272 } 273 274 /* 275 * buf_unmapped() default callback. Does nothing. 276 */ 277 static void buf_unmapped_default_callback(struct rchan_buf *buf, 278 struct file *filp) 279 { 280 } 281 282 /* 283 * create_buf_file_create() default callback. Does nothing. 284 */ 285 static struct dentry *create_buf_file_default_callback(const char *filename, 286 struct dentry *parent, 287 int mode, 288 struct rchan_buf *buf, 289 int *is_global) 290 { 291 return NULL; 292 } 293 294 /* 295 * remove_buf_file() default callback. Does nothing. 296 */ 297 static int remove_buf_file_default_callback(struct dentry *dentry) 298 { 299 return -EINVAL; 300 } 301 302 /* relay channel default callbacks */ 303 static struct rchan_callbacks default_channel_callbacks = { 304 .subbuf_start = subbuf_start_default_callback, 305 .buf_mapped = buf_mapped_default_callback, 306 .buf_unmapped = buf_unmapped_default_callback, 307 .create_buf_file = create_buf_file_default_callback, 308 .remove_buf_file = remove_buf_file_default_callback, 309 }; 310 311 /** 312 * wakeup_readers - wake up readers waiting on a channel 313 * @data: contains the channel buffer 314 * 315 * This is the timer function used to defer reader waking. 316 */ 317 static void wakeup_readers(unsigned long data) 318 { 319 struct rchan_buf *buf = (struct rchan_buf *)data; 320 wake_up_interruptible(&buf->read_wait); 321 } 322 323 /** 324 * __relay_reset - reset a channel buffer 325 * @buf: the channel buffer 326 * @init: 1 if this is a first-time initialization 327 * 328 * See relay_reset() for description of effect. 329 */ 330 static void __relay_reset(struct rchan_buf *buf, unsigned int init) 331 { 332 size_t i; 333 334 if (init) { 335 init_waitqueue_head(&buf->read_wait); 336 kref_init(&buf->kref); 337 setup_timer(&buf->timer, wakeup_readers, (unsigned long)buf); 338 } else 339 del_timer_sync(&buf->timer); 340 341 buf->subbufs_produced = 0; 342 buf->subbufs_consumed = 0; 343 buf->bytes_consumed = 0; 344 buf->finalized = 0; 345 buf->data = buf->start; 346 buf->offset = 0; 347 348 for (i = 0; i < buf->chan->n_subbufs; i++) 349 buf->padding[i] = 0; 350 351 buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0); 352 } 353 354 /** 355 * relay_reset - reset the channel 356 * @chan: the channel 357 * 358 * This has the effect of erasing all data from all channel buffers 359 * and restarting the channel in its initial state. The buffers 360 * are not freed, so any mappings are still in effect. 361 * 362 * NOTE. Care should be taken that the channel isn't actually 363 * being used by anything when this call is made. 364 */ 365 void relay_reset(struct rchan *chan) 366 { 367 unsigned int i; 368 369 if (!chan) 370 return; 371 372 if (chan->is_global && chan->buf[0]) { 373 __relay_reset(chan->buf[0], 0); 374 return; 375 } 376 377 mutex_lock(&relay_channels_mutex); 378 for_each_online_cpu(i) 379 if (chan->buf[i]) 380 __relay_reset(chan->buf[i], 0); 381 mutex_unlock(&relay_channels_mutex); 382 } 383 EXPORT_SYMBOL_GPL(relay_reset); 384 385 /* 386 * relay_open_buf - create a new relay channel buffer 387 * 388 * used by relay_open() and CPU hotplug. 389 */ 390 static struct rchan_buf *relay_open_buf(struct rchan *chan, unsigned int cpu) 391 { 392 struct rchan_buf *buf = NULL; 393 struct dentry *dentry; 394 char *tmpname; 395 396 if (chan->is_global) 397 return chan->buf[0]; 398 399 tmpname = kzalloc(NAME_MAX + 1, GFP_KERNEL); 400 if (!tmpname) 401 goto end; 402 snprintf(tmpname, NAME_MAX, "%s%d", chan->base_filename, cpu); 403 404 buf = relay_create_buf(chan); 405 if (!buf) 406 goto free_name; 407 408 buf->cpu = cpu; 409 __relay_reset(buf, 1); 410 411 /* Create file in fs */ 412 dentry = chan->cb->create_buf_file(tmpname, chan->parent, S_IRUSR, 413 buf, &chan->is_global); 414 if (!dentry) 415 goto free_buf; 416 417 buf->dentry = dentry; 418 419 if(chan->is_global) { 420 chan->buf[0] = buf; 421 buf->cpu = 0; 422 } 423 424 goto free_name; 425 426 free_buf: 427 relay_destroy_buf(buf); 428 free_name: 429 kfree(tmpname); 430 end: 431 return buf; 432 } 433 434 /** 435 * relay_close_buf - close a channel buffer 436 * @buf: channel buffer 437 * 438 * Marks the buffer finalized and restores the default callbacks. 439 * The channel buffer and channel buffer data structure are then freed 440 * automatically when the last reference is given up. 441 */ 442 static void relay_close_buf(struct rchan_buf *buf) 443 { 444 buf->finalized = 1; 445 del_timer_sync(&buf->timer); 446 kref_put(&buf->kref, relay_remove_buf); 447 } 448 449 static void setup_callbacks(struct rchan *chan, 450 struct rchan_callbacks *cb) 451 { 452 if (!cb) { 453 chan->cb = &default_channel_callbacks; 454 return; 455 } 456 457 if (!cb->subbuf_start) 458 cb->subbuf_start = subbuf_start_default_callback; 459 if (!cb->buf_mapped) 460 cb->buf_mapped = buf_mapped_default_callback; 461 if (!cb->buf_unmapped) 462 cb->buf_unmapped = buf_unmapped_default_callback; 463 if (!cb->create_buf_file) 464 cb->create_buf_file = create_buf_file_default_callback; 465 if (!cb->remove_buf_file) 466 cb->remove_buf_file = remove_buf_file_default_callback; 467 chan->cb = cb; 468 } 469 470 /** 471 * relay_hotcpu_callback - CPU hotplug callback 472 * @nb: notifier block 473 * @action: hotplug action to take 474 * @hcpu: CPU number 475 * 476 * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD) 477 */ 478 static int __cpuinit relay_hotcpu_callback(struct notifier_block *nb, 479 unsigned long action, 480 void *hcpu) 481 { 482 unsigned int hotcpu = (unsigned long)hcpu; 483 struct rchan *chan; 484 485 switch(action) { 486 case CPU_UP_PREPARE: 487 case CPU_UP_PREPARE_FROZEN: 488 mutex_lock(&relay_channels_mutex); 489 list_for_each_entry(chan, &relay_channels, list) { 490 if (chan->buf[hotcpu]) 491 continue; 492 chan->buf[hotcpu] = relay_open_buf(chan, hotcpu); 493 if(!chan->buf[hotcpu]) { 494 printk(KERN_ERR 495 "relay_hotcpu_callback: cpu %d buffer " 496 "creation failed\n", hotcpu); 497 mutex_unlock(&relay_channels_mutex); 498 return NOTIFY_BAD; 499 } 500 } 501 mutex_unlock(&relay_channels_mutex); 502 break; 503 case CPU_DEAD: 504 case CPU_DEAD_FROZEN: 505 /* No need to flush the cpu : will be flushed upon 506 * final relay_flush() call. */ 507 break; 508 } 509 return NOTIFY_OK; 510 } 511 512 /** 513 * relay_open - create a new relay channel 514 * @base_filename: base name of files to create 515 * @parent: dentry of parent directory, %NULL for root directory 516 * @subbuf_size: size of sub-buffers 517 * @n_subbufs: number of sub-buffers 518 * @cb: client callback functions 519 * @private_data: user-defined data 520 * 521 * Returns channel pointer if successful, %NULL otherwise. 522 * 523 * Creates a channel buffer for each cpu using the sizes and 524 * attributes specified. The created channel buffer files 525 * will be named base_filename0...base_filenameN-1. File 526 * permissions will be %S_IRUSR. 527 */ 528 struct rchan *relay_open(const char *base_filename, 529 struct dentry *parent, 530 size_t subbuf_size, 531 size_t n_subbufs, 532 struct rchan_callbacks *cb, 533 void *private_data) 534 { 535 unsigned int i; 536 struct rchan *chan; 537 if (!base_filename) 538 return NULL; 539 540 if (!(subbuf_size && n_subbufs)) 541 return NULL; 542 543 chan = kzalloc(sizeof(struct rchan), GFP_KERNEL); 544 if (!chan) 545 return NULL; 546 547 chan->version = RELAYFS_CHANNEL_VERSION; 548 chan->n_subbufs = n_subbufs; 549 chan->subbuf_size = subbuf_size; 550 chan->alloc_size = FIX_SIZE(subbuf_size * n_subbufs); 551 chan->parent = parent; 552 chan->private_data = private_data; 553 strlcpy(chan->base_filename, base_filename, NAME_MAX); 554 setup_callbacks(chan, cb); 555 kref_init(&chan->kref); 556 557 mutex_lock(&relay_channels_mutex); 558 for_each_online_cpu(i) { 559 chan->buf[i] = relay_open_buf(chan, i); 560 if (!chan->buf[i]) 561 goto free_bufs; 562 } 563 list_add(&chan->list, &relay_channels); 564 mutex_unlock(&relay_channels_mutex); 565 566 return chan; 567 568 free_bufs: 569 for_each_online_cpu(i) { 570 if (!chan->buf[i]) 571 break; 572 relay_close_buf(chan->buf[i]); 573 } 574 575 kref_put(&chan->kref, relay_destroy_channel); 576 mutex_unlock(&relay_channels_mutex); 577 return NULL; 578 } 579 EXPORT_SYMBOL_GPL(relay_open); 580 581 /** 582 * relay_switch_subbuf - switch to a new sub-buffer 583 * @buf: channel buffer 584 * @length: size of current event 585 * 586 * Returns either the length passed in or 0 if full. 587 * 588 * Performs sub-buffer-switch tasks such as invoking callbacks, 589 * updating padding counts, waking up readers, etc. 590 */ 591 size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length) 592 { 593 void *old, *new; 594 size_t old_subbuf, new_subbuf; 595 596 if (unlikely(length > buf->chan->subbuf_size)) 597 goto toobig; 598 599 if (buf->offset != buf->chan->subbuf_size + 1) { 600 buf->prev_padding = buf->chan->subbuf_size - buf->offset; 601 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; 602 buf->padding[old_subbuf] = buf->prev_padding; 603 buf->subbufs_produced++; 604 buf->dentry->d_inode->i_size += buf->chan->subbuf_size - 605 buf->padding[old_subbuf]; 606 smp_mb(); 607 if (waitqueue_active(&buf->read_wait)) 608 /* 609 * Calling wake_up_interruptible() from here 610 * will deadlock if we happen to be logging 611 * from the scheduler (trying to re-grab 612 * rq->lock), so defer it. 613 */ 614 __mod_timer(&buf->timer, jiffies + 1); 615 } 616 617 old = buf->data; 618 new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; 619 new = buf->start + new_subbuf * buf->chan->subbuf_size; 620 buf->offset = 0; 621 if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) { 622 buf->offset = buf->chan->subbuf_size + 1; 623 return 0; 624 } 625 buf->data = new; 626 buf->padding[new_subbuf] = 0; 627 628 if (unlikely(length + buf->offset > buf->chan->subbuf_size)) 629 goto toobig; 630 631 return length; 632 633 toobig: 634 buf->chan->last_toobig = length; 635 return 0; 636 } 637 EXPORT_SYMBOL_GPL(relay_switch_subbuf); 638 639 /** 640 * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count 641 * @chan: the channel 642 * @cpu: the cpu associated with the channel buffer to update 643 * @subbufs_consumed: number of sub-buffers to add to current buf's count 644 * 645 * Adds to the channel buffer's consumed sub-buffer count. 646 * subbufs_consumed should be the number of sub-buffers newly consumed, 647 * not the total consumed. 648 * 649 * NOTE. Kernel clients don't need to call this function if the channel 650 * mode is 'overwrite'. 651 */ 652 void relay_subbufs_consumed(struct rchan *chan, 653 unsigned int cpu, 654 size_t subbufs_consumed) 655 { 656 struct rchan_buf *buf; 657 658 if (!chan) 659 return; 660 661 if (cpu >= NR_CPUS || !chan->buf[cpu]) 662 return; 663 664 buf = chan->buf[cpu]; 665 buf->subbufs_consumed += subbufs_consumed; 666 if (buf->subbufs_consumed > buf->subbufs_produced) 667 buf->subbufs_consumed = buf->subbufs_produced; 668 } 669 EXPORT_SYMBOL_GPL(relay_subbufs_consumed); 670 671 /** 672 * relay_close - close the channel 673 * @chan: the channel 674 * 675 * Closes all channel buffers and frees the channel. 676 */ 677 void relay_close(struct rchan *chan) 678 { 679 unsigned int i; 680 681 if (!chan) 682 return; 683 684 mutex_lock(&relay_channels_mutex); 685 if (chan->is_global && chan->buf[0]) 686 relay_close_buf(chan->buf[0]); 687 else 688 for_each_possible_cpu(i) 689 if (chan->buf[i]) 690 relay_close_buf(chan->buf[i]); 691 692 if (chan->last_toobig) 693 printk(KERN_WARNING "relay: one or more items not logged " 694 "[item size (%Zd) > sub-buffer size (%Zd)]\n", 695 chan->last_toobig, chan->subbuf_size); 696 697 list_del(&chan->list); 698 kref_put(&chan->kref, relay_destroy_channel); 699 mutex_unlock(&relay_channels_mutex); 700 } 701 EXPORT_SYMBOL_GPL(relay_close); 702 703 /** 704 * relay_flush - close the channel 705 * @chan: the channel 706 * 707 * Flushes all channel buffers, i.e. forces buffer switch. 708 */ 709 void relay_flush(struct rchan *chan) 710 { 711 unsigned int i; 712 713 if (!chan) 714 return; 715 716 if (chan->is_global && chan->buf[0]) { 717 relay_switch_subbuf(chan->buf[0], 0); 718 return; 719 } 720 721 mutex_lock(&relay_channels_mutex); 722 for_each_possible_cpu(i) 723 if (chan->buf[i]) 724 relay_switch_subbuf(chan->buf[i], 0); 725 mutex_unlock(&relay_channels_mutex); 726 } 727 EXPORT_SYMBOL_GPL(relay_flush); 728 729 /** 730 * relay_file_open - open file op for relay files 731 * @inode: the inode 732 * @filp: the file 733 * 734 * Increments the channel buffer refcount. 735 */ 736 static int relay_file_open(struct inode *inode, struct file *filp) 737 { 738 struct rchan_buf *buf = inode->i_private; 739 kref_get(&buf->kref); 740 filp->private_data = buf; 741 742 return 0; 743 } 744 745 /** 746 * relay_file_mmap - mmap file op for relay files 747 * @filp: the file 748 * @vma: the vma describing what to map 749 * 750 * Calls upon relay_mmap_buf() to map the file into user space. 751 */ 752 static int relay_file_mmap(struct file *filp, struct vm_area_struct *vma) 753 { 754 struct rchan_buf *buf = filp->private_data; 755 return relay_mmap_buf(buf, vma); 756 } 757 758 /** 759 * relay_file_poll - poll file op for relay files 760 * @filp: the file 761 * @wait: poll table 762 * 763 * Poll implemention. 764 */ 765 static unsigned int relay_file_poll(struct file *filp, poll_table *wait) 766 { 767 unsigned int mask = 0; 768 struct rchan_buf *buf = filp->private_data; 769 770 if (buf->finalized) 771 return POLLERR; 772 773 if (filp->f_mode & FMODE_READ) { 774 poll_wait(filp, &buf->read_wait, wait); 775 if (!relay_buf_empty(buf)) 776 mask |= POLLIN | POLLRDNORM; 777 } 778 779 return mask; 780 } 781 782 /** 783 * relay_file_release - release file op for relay files 784 * @inode: the inode 785 * @filp: the file 786 * 787 * Decrements the channel refcount, as the filesystem is 788 * no longer using it. 789 */ 790 static int relay_file_release(struct inode *inode, struct file *filp) 791 { 792 struct rchan_buf *buf = filp->private_data; 793 kref_put(&buf->kref, relay_remove_buf); 794 795 return 0; 796 } 797 798 /* 799 * relay_file_read_consume - update the consumed count for the buffer 800 */ 801 static void relay_file_read_consume(struct rchan_buf *buf, 802 size_t read_pos, 803 size_t bytes_consumed) 804 { 805 size_t subbuf_size = buf->chan->subbuf_size; 806 size_t n_subbufs = buf->chan->n_subbufs; 807 size_t read_subbuf; 808 809 if (buf->bytes_consumed + bytes_consumed > subbuf_size) { 810 relay_subbufs_consumed(buf->chan, buf->cpu, 1); 811 buf->bytes_consumed = 0; 812 } 813 814 buf->bytes_consumed += bytes_consumed; 815 read_subbuf = read_pos / buf->chan->subbuf_size; 816 if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) { 817 if ((read_subbuf == buf->subbufs_produced % n_subbufs) && 818 (buf->offset == subbuf_size)) 819 return; 820 relay_subbufs_consumed(buf->chan, buf->cpu, 1); 821 buf->bytes_consumed = 0; 822 } 823 } 824 825 /* 826 * relay_file_read_avail - boolean, are there unconsumed bytes available? 827 */ 828 static int relay_file_read_avail(struct rchan_buf *buf, size_t read_pos) 829 { 830 size_t subbuf_size = buf->chan->subbuf_size; 831 size_t n_subbufs = buf->chan->n_subbufs; 832 size_t produced = buf->subbufs_produced; 833 size_t consumed = buf->subbufs_consumed; 834 835 relay_file_read_consume(buf, read_pos, 0); 836 837 if (unlikely(buf->offset > subbuf_size)) { 838 if (produced == consumed) 839 return 0; 840 return 1; 841 } 842 843 if (unlikely(produced - consumed >= n_subbufs)) { 844 consumed = (produced / n_subbufs) * n_subbufs; 845 buf->subbufs_consumed = consumed; 846 } 847 848 produced = (produced % n_subbufs) * subbuf_size + buf->offset; 849 consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed; 850 851 if (consumed > produced) 852 produced += n_subbufs * subbuf_size; 853 854 if (consumed == produced) 855 return 0; 856 857 return 1; 858 } 859 860 /** 861 * relay_file_read_subbuf_avail - return bytes available in sub-buffer 862 * @read_pos: file read position 863 * @buf: relay channel buffer 864 */ 865 static size_t relay_file_read_subbuf_avail(size_t read_pos, 866 struct rchan_buf *buf) 867 { 868 size_t padding, avail = 0; 869 size_t read_subbuf, read_offset, write_subbuf, write_offset; 870 size_t subbuf_size = buf->chan->subbuf_size; 871 872 write_subbuf = (buf->data - buf->start) / subbuf_size; 873 write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset; 874 read_subbuf = read_pos / subbuf_size; 875 read_offset = read_pos % subbuf_size; 876 padding = buf->padding[read_subbuf]; 877 878 if (read_subbuf == write_subbuf) { 879 if (read_offset + padding < write_offset) 880 avail = write_offset - (read_offset + padding); 881 } else 882 avail = (subbuf_size - padding) - read_offset; 883 884 return avail; 885 } 886 887 /** 888 * relay_file_read_start_pos - find the first available byte to read 889 * @read_pos: file read position 890 * @buf: relay channel buffer 891 * 892 * If the @read_pos is in the middle of padding, return the 893 * position of the first actually available byte, otherwise 894 * return the original value. 895 */ 896 static size_t relay_file_read_start_pos(size_t read_pos, 897 struct rchan_buf *buf) 898 { 899 size_t read_subbuf, padding, padding_start, padding_end; 900 size_t subbuf_size = buf->chan->subbuf_size; 901 size_t n_subbufs = buf->chan->n_subbufs; 902 903 read_subbuf = read_pos / subbuf_size; 904 padding = buf->padding[read_subbuf]; 905 padding_start = (read_subbuf + 1) * subbuf_size - padding; 906 padding_end = (read_subbuf + 1) * subbuf_size; 907 if (read_pos >= padding_start && read_pos < padding_end) { 908 read_subbuf = (read_subbuf + 1) % n_subbufs; 909 read_pos = read_subbuf * subbuf_size; 910 } 911 912 return read_pos; 913 } 914 915 /** 916 * relay_file_read_end_pos - return the new read position 917 * @read_pos: file read position 918 * @buf: relay channel buffer 919 * @count: number of bytes to be read 920 */ 921 static size_t relay_file_read_end_pos(struct rchan_buf *buf, 922 size_t read_pos, 923 size_t count) 924 { 925 size_t read_subbuf, padding, end_pos; 926 size_t subbuf_size = buf->chan->subbuf_size; 927 size_t n_subbufs = buf->chan->n_subbufs; 928 929 read_subbuf = read_pos / subbuf_size; 930 padding = buf->padding[read_subbuf]; 931 if (read_pos % subbuf_size + count + padding == subbuf_size) 932 end_pos = (read_subbuf + 1) * subbuf_size; 933 else 934 end_pos = read_pos + count; 935 if (end_pos >= subbuf_size * n_subbufs) 936 end_pos = 0; 937 938 return end_pos; 939 } 940 941 /* 942 * subbuf_read_actor - read up to one subbuf's worth of data 943 */ 944 static int subbuf_read_actor(size_t read_start, 945 struct rchan_buf *buf, 946 size_t avail, 947 read_descriptor_t *desc, 948 read_actor_t actor) 949 { 950 void *from; 951 int ret = 0; 952 953 from = buf->start + read_start; 954 ret = avail; 955 if (copy_to_user(desc->arg.buf, from, avail)) { 956 desc->error = -EFAULT; 957 ret = 0; 958 } 959 desc->arg.data += ret; 960 desc->written += ret; 961 desc->count -= ret; 962 963 return ret; 964 } 965 966 /* 967 * subbuf_send_actor - send up to one subbuf's worth of data 968 */ 969 static int subbuf_send_actor(size_t read_start, 970 struct rchan_buf *buf, 971 size_t avail, 972 read_descriptor_t *desc, 973 read_actor_t actor) 974 { 975 unsigned long pidx, poff; 976 unsigned int subbuf_pages; 977 int ret = 0; 978 979 subbuf_pages = buf->chan->alloc_size >> PAGE_SHIFT; 980 pidx = (read_start / PAGE_SIZE) % subbuf_pages; 981 poff = read_start & ~PAGE_MASK; 982 while (avail) { 983 struct page *p = buf->page_array[pidx]; 984 unsigned int len; 985 986 len = PAGE_SIZE - poff; 987 if (len > avail) 988 len = avail; 989 990 len = actor(desc, p, poff, len); 991 if (desc->error) 992 break; 993 994 avail -= len; 995 ret += len; 996 poff = 0; 997 pidx = (pidx + 1) % subbuf_pages; 998 } 999 1000 return ret; 1001 } 1002 1003 typedef int (*subbuf_actor_t) (size_t read_start, 1004 struct rchan_buf *buf, 1005 size_t avail, 1006 read_descriptor_t *desc, 1007 read_actor_t actor); 1008 1009 /* 1010 * relay_file_read_subbufs - read count bytes, bridging subbuf boundaries 1011 */ 1012 static ssize_t relay_file_read_subbufs(struct file *filp, loff_t *ppos, 1013 subbuf_actor_t subbuf_actor, 1014 read_actor_t actor, 1015 read_descriptor_t *desc) 1016 { 1017 struct rchan_buf *buf = filp->private_data; 1018 size_t read_start, avail; 1019 int ret; 1020 1021 if (!desc->count) 1022 return 0; 1023 1024 mutex_lock(&filp->f_path.dentry->d_inode->i_mutex); 1025 do { 1026 if (!relay_file_read_avail(buf, *ppos)) 1027 break; 1028 1029 read_start = relay_file_read_start_pos(*ppos, buf); 1030 avail = relay_file_read_subbuf_avail(read_start, buf); 1031 if (!avail) 1032 break; 1033 1034 avail = min(desc->count, avail); 1035 ret = subbuf_actor(read_start, buf, avail, desc, actor); 1036 if (desc->error < 0) 1037 break; 1038 1039 if (ret) { 1040 relay_file_read_consume(buf, read_start, ret); 1041 *ppos = relay_file_read_end_pos(buf, read_start, ret); 1042 } 1043 } while (desc->count && ret); 1044 mutex_unlock(&filp->f_path.dentry->d_inode->i_mutex); 1045 1046 return desc->written; 1047 } 1048 1049 static ssize_t relay_file_read(struct file *filp, 1050 char __user *buffer, 1051 size_t count, 1052 loff_t *ppos) 1053 { 1054 read_descriptor_t desc; 1055 desc.written = 0; 1056 desc.count = count; 1057 desc.arg.buf = buffer; 1058 desc.error = 0; 1059 return relay_file_read_subbufs(filp, ppos, subbuf_read_actor, 1060 NULL, &desc); 1061 } 1062 1063 static ssize_t relay_file_sendfile(struct file *filp, 1064 loff_t *ppos, 1065 size_t count, 1066 read_actor_t actor, 1067 void *target) 1068 { 1069 read_descriptor_t desc; 1070 desc.written = 0; 1071 desc.count = count; 1072 desc.arg.data = target; 1073 desc.error = 0; 1074 return relay_file_read_subbufs(filp, ppos, subbuf_send_actor, 1075 actor, &desc); 1076 } 1077 1078 const struct file_operations relay_file_operations = { 1079 .open = relay_file_open, 1080 .poll = relay_file_poll, 1081 .mmap = relay_file_mmap, 1082 .read = relay_file_read, 1083 .llseek = no_llseek, 1084 .release = relay_file_release, 1085 .sendfile = relay_file_sendfile, 1086 }; 1087 EXPORT_SYMBOL_GPL(relay_file_operations); 1088 1089 static __init int relay_init(void) 1090 { 1091 1092 hotcpu_notifier(relay_hotcpu_callback, 0); 1093 return 0; 1094 } 1095 1096 module_init(relay_init); 1097