1 /* 2 * Tty buffer allocation management 3 */ 4 5 #include <linux/types.h> 6 #include <linux/errno.h> 7 #include <linux/tty.h> 8 #include <linux/tty_driver.h> 9 #include <linux/tty_flip.h> 10 #include <linux/timer.h> 11 #include <linux/string.h> 12 #include <linux/slab.h> 13 #include <linux/sched.h> 14 #include <linux/wait.h> 15 #include <linux/bitops.h> 16 #include <linux/delay.h> 17 #include <linux/module.h> 18 #include <linux/ratelimit.h> 19 20 21 #define MIN_TTYB_SIZE 256 22 #define TTYB_ALIGN_MASK 255 23 24 /* 25 * Byte threshold to limit memory consumption for flip buffers. 26 * The actual memory limit is > 2x this amount. 27 */ 28 #define TTYB_DEFAULT_MEM_LIMIT 65536 29 30 /* 31 * We default to dicing tty buffer allocations to this many characters 32 * in order to avoid multiple page allocations. We know the size of 33 * tty_buffer itself but it must also be taken into account that the 34 * the buffer is 256 byte aligned. See tty_buffer_find for the allocation 35 * logic this must match 36 */ 37 38 #define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~0xFF) 39 40 /* 41 * If all tty flip buffers have been processed by flush_to_ldisc() or 42 * dropped by tty_buffer_flush(), check if the linked pty has been closed. 43 * If so, wake the reader/poll to process 44 */ 45 static inline void check_other_closed(struct tty_struct *tty) 46 { 47 unsigned long flags, old; 48 49 /* transition from TTY_OTHER_CLOSED => TTY_OTHER_DONE must be atomic */ 50 for (flags = ACCESS_ONCE(tty->flags); 51 test_bit(TTY_OTHER_CLOSED, &flags); 52 ) { 53 old = flags; 54 __set_bit(TTY_OTHER_DONE, &flags); 55 flags = cmpxchg(&tty->flags, old, flags); 56 if (old == flags) { 57 wake_up_interruptible(&tty->read_wait); 58 break; 59 } 60 } 61 } 62 63 /** 64 * tty_buffer_lock_exclusive - gain exclusive access to buffer 65 * tty_buffer_unlock_exclusive - release exclusive access 66 * 67 * @port - tty_port owning the flip buffer 68 * 69 * Guarantees safe use of the line discipline's receive_buf() method by 70 * excluding the buffer work and any pending flush from using the flip 71 * buffer. Data can continue to be added concurrently to the flip buffer 72 * from the driver side. 73 * 74 * On release, the buffer work is restarted if there is data in the 75 * flip buffer 76 */ 77 78 void tty_buffer_lock_exclusive(struct tty_port *port) 79 { 80 struct tty_bufhead *buf = &port->buf; 81 82 atomic_inc(&buf->priority); 83 mutex_lock(&buf->lock); 84 } 85 EXPORT_SYMBOL_GPL(tty_buffer_lock_exclusive); 86 87 void tty_buffer_unlock_exclusive(struct tty_port *port) 88 { 89 struct tty_bufhead *buf = &port->buf; 90 int restart; 91 92 restart = buf->head->commit != buf->head->read; 93 94 atomic_dec(&buf->priority); 95 mutex_unlock(&buf->lock); 96 if (restart) 97 queue_work(system_unbound_wq, &buf->work); 98 } 99 EXPORT_SYMBOL_GPL(tty_buffer_unlock_exclusive); 100 101 /** 102 * tty_buffer_space_avail - return unused buffer space 103 * @port - tty_port owning the flip buffer 104 * 105 * Returns the # of bytes which can be written by the driver without 106 * reaching the buffer limit. 107 * 108 * Note: this does not guarantee that memory is available to write 109 * the returned # of bytes (use tty_prepare_flip_string_xxx() to 110 * pre-allocate if memory guarantee is required). 111 */ 112 113 int tty_buffer_space_avail(struct tty_port *port) 114 { 115 int space = port->buf.mem_limit - atomic_read(&port->buf.mem_used); 116 return max(space, 0); 117 } 118 EXPORT_SYMBOL_GPL(tty_buffer_space_avail); 119 120 static void tty_buffer_reset(struct tty_buffer *p, size_t size) 121 { 122 p->used = 0; 123 p->size = size; 124 p->next = NULL; 125 p->commit = 0; 126 p->read = 0; 127 p->flags = 0; 128 } 129 130 /** 131 * tty_buffer_free_all - free buffers used by a tty 132 * @tty: tty to free from 133 * 134 * Remove all the buffers pending on a tty whether queued with data 135 * or in the free ring. Must be called when the tty is no longer in use 136 */ 137 138 void tty_buffer_free_all(struct tty_port *port) 139 { 140 struct tty_bufhead *buf = &port->buf; 141 struct tty_buffer *p, *next; 142 struct llist_node *llist; 143 144 while ((p = buf->head) != NULL) { 145 buf->head = p->next; 146 if (p->size > 0) 147 kfree(p); 148 } 149 llist = llist_del_all(&buf->free); 150 llist_for_each_entry_safe(p, next, llist, free) 151 kfree(p); 152 153 tty_buffer_reset(&buf->sentinel, 0); 154 buf->head = &buf->sentinel; 155 buf->tail = &buf->sentinel; 156 157 atomic_set(&buf->mem_used, 0); 158 } 159 160 /** 161 * tty_buffer_alloc - allocate a tty buffer 162 * @tty: tty device 163 * @size: desired size (characters) 164 * 165 * Allocate a new tty buffer to hold the desired number of characters. 166 * We round our buffers off in 256 character chunks to get better 167 * allocation behaviour. 168 * Return NULL if out of memory or the allocation would exceed the 169 * per device queue 170 */ 171 172 static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size) 173 { 174 struct llist_node *free; 175 struct tty_buffer *p; 176 177 /* Round the buffer size out */ 178 size = __ALIGN_MASK(size, TTYB_ALIGN_MASK); 179 180 if (size <= MIN_TTYB_SIZE) { 181 free = llist_del_first(&port->buf.free); 182 if (free) { 183 p = llist_entry(free, struct tty_buffer, free); 184 goto found; 185 } 186 } 187 188 /* Should possibly check if this fails for the largest buffer we 189 have queued and recycle that ? */ 190 if (atomic_read(&port->buf.mem_used) > port->buf.mem_limit) 191 return NULL; 192 p = kmalloc(sizeof(struct tty_buffer) + 2 * size, GFP_ATOMIC); 193 if (p == NULL) 194 return NULL; 195 196 found: 197 tty_buffer_reset(p, size); 198 atomic_add(size, &port->buf.mem_used); 199 return p; 200 } 201 202 /** 203 * tty_buffer_free - free a tty buffer 204 * @tty: tty owning the buffer 205 * @b: the buffer to free 206 * 207 * Free a tty buffer, or add it to the free list according to our 208 * internal strategy 209 */ 210 211 static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b) 212 { 213 struct tty_bufhead *buf = &port->buf; 214 215 /* Dumb strategy for now - should keep some stats */ 216 WARN_ON(atomic_sub_return(b->size, &buf->mem_used) < 0); 217 218 if (b->size > MIN_TTYB_SIZE) 219 kfree(b); 220 else if (b->size > 0) 221 llist_add(&b->free, &buf->free); 222 } 223 224 /** 225 * tty_buffer_flush - flush full tty buffers 226 * @tty: tty to flush 227 * @ld: optional ldisc ptr (must be referenced) 228 * 229 * flush all the buffers containing receive data. If ld != NULL, 230 * flush the ldisc input buffer. 231 * 232 * Locking: takes buffer lock to ensure single-threaded flip buffer 233 * 'consumer' 234 */ 235 236 void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld) 237 { 238 struct tty_port *port = tty->port; 239 struct tty_bufhead *buf = &port->buf; 240 struct tty_buffer *next; 241 242 atomic_inc(&buf->priority); 243 244 mutex_lock(&buf->lock); 245 /* paired w/ release in __tty_buffer_request_room; ensures there are 246 * no pending memory accesses to the freed buffer 247 */ 248 while ((next = smp_load_acquire(&buf->head->next)) != NULL) { 249 tty_buffer_free(port, buf->head); 250 buf->head = next; 251 } 252 buf->head->read = buf->head->commit; 253 254 if (ld && ld->ops->flush_buffer) 255 ld->ops->flush_buffer(tty); 256 257 check_other_closed(tty); 258 259 atomic_dec(&buf->priority); 260 mutex_unlock(&buf->lock); 261 } 262 263 /** 264 * tty_buffer_request_room - grow tty buffer if needed 265 * @tty: tty structure 266 * @size: size desired 267 * @flags: buffer flags if new buffer allocated (default = 0) 268 * 269 * Make at least size bytes of linear space available for the tty 270 * buffer. If we fail return the size we managed to find. 271 * 272 * Will change over to a new buffer if the current buffer is encoded as 273 * TTY_NORMAL (so has no flags buffer) and the new buffer requires 274 * a flags buffer. 275 */ 276 static int __tty_buffer_request_room(struct tty_port *port, size_t size, 277 int flags) 278 { 279 struct tty_bufhead *buf = &port->buf; 280 struct tty_buffer *b, *n; 281 int left, change; 282 283 b = buf->tail; 284 if (b->flags & TTYB_NORMAL) 285 left = 2 * b->size - b->used; 286 else 287 left = b->size - b->used; 288 289 change = (b->flags & TTYB_NORMAL) && (~flags & TTYB_NORMAL); 290 if (change || left < size) { 291 /* This is the slow path - looking for new buffers to use */ 292 n = tty_buffer_alloc(port, size); 293 if (n != NULL) { 294 n->flags = flags; 295 buf->tail = n; 296 /* paired w/ acquire in flush_to_ldisc(); ensures 297 * flush_to_ldisc() sees buffer data. 298 */ 299 smp_store_release(&b->commit, b->used); 300 /* paired w/ acquire in flush_to_ldisc(); ensures the 301 * latest commit value can be read before the head is 302 * advanced to the next buffer 303 */ 304 smp_store_release(&b->next, n); 305 } else if (change) 306 size = 0; 307 else 308 size = left; 309 } 310 return size; 311 } 312 313 int tty_buffer_request_room(struct tty_port *port, size_t size) 314 { 315 return __tty_buffer_request_room(port, size, 0); 316 } 317 EXPORT_SYMBOL_GPL(tty_buffer_request_room); 318 319 /** 320 * tty_insert_flip_string_fixed_flag - Add characters to the tty buffer 321 * @port: tty port 322 * @chars: characters 323 * @flag: flag value for each character 324 * @size: size 325 * 326 * Queue a series of bytes to the tty buffering. All the characters 327 * passed are marked with the supplied flag. Returns the number added. 328 */ 329 330 int tty_insert_flip_string_fixed_flag(struct tty_port *port, 331 const unsigned char *chars, char flag, size_t size) 332 { 333 int copied = 0; 334 do { 335 int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE); 336 int flags = (flag == TTY_NORMAL) ? TTYB_NORMAL : 0; 337 int space = __tty_buffer_request_room(port, goal, flags); 338 struct tty_buffer *tb = port->buf.tail; 339 if (unlikely(space == 0)) 340 break; 341 memcpy(char_buf_ptr(tb, tb->used), chars, space); 342 if (~tb->flags & TTYB_NORMAL) 343 memset(flag_buf_ptr(tb, tb->used), flag, space); 344 tb->used += space; 345 copied += space; 346 chars += space; 347 /* There is a small chance that we need to split the data over 348 several buffers. If this is the case we must loop */ 349 } while (unlikely(size > copied)); 350 return copied; 351 } 352 EXPORT_SYMBOL(tty_insert_flip_string_fixed_flag); 353 354 /** 355 * tty_insert_flip_string_flags - Add characters to the tty buffer 356 * @port: tty port 357 * @chars: characters 358 * @flags: flag bytes 359 * @size: size 360 * 361 * Queue a series of bytes to the tty buffering. For each character 362 * the flags array indicates the status of the character. Returns the 363 * number added. 364 */ 365 366 int tty_insert_flip_string_flags(struct tty_port *port, 367 const unsigned char *chars, const char *flags, size_t size) 368 { 369 int copied = 0; 370 do { 371 int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE); 372 int space = tty_buffer_request_room(port, goal); 373 struct tty_buffer *tb = port->buf.tail; 374 if (unlikely(space == 0)) 375 break; 376 memcpy(char_buf_ptr(tb, tb->used), chars, space); 377 memcpy(flag_buf_ptr(tb, tb->used), flags, space); 378 tb->used += space; 379 copied += space; 380 chars += space; 381 flags += space; 382 /* There is a small chance that we need to split the data over 383 several buffers. If this is the case we must loop */ 384 } while (unlikely(size > copied)); 385 return copied; 386 } 387 EXPORT_SYMBOL(tty_insert_flip_string_flags); 388 389 /** 390 * tty_schedule_flip - push characters to ldisc 391 * @port: tty port to push from 392 * 393 * Takes any pending buffers and transfers their ownership to the 394 * ldisc side of the queue. It then schedules those characters for 395 * processing by the line discipline. 396 */ 397 398 void tty_schedule_flip(struct tty_port *port) 399 { 400 struct tty_bufhead *buf = &port->buf; 401 402 /* paired w/ acquire in flush_to_ldisc(); ensures 403 * flush_to_ldisc() sees buffer data. 404 */ 405 smp_store_release(&buf->tail->commit, buf->tail->used); 406 queue_work(system_unbound_wq, &buf->work); 407 } 408 EXPORT_SYMBOL(tty_schedule_flip); 409 410 /** 411 * tty_prepare_flip_string - make room for characters 412 * @port: tty port 413 * @chars: return pointer for character write area 414 * @size: desired size 415 * 416 * Prepare a block of space in the buffer for data. Returns the length 417 * available and buffer pointer to the space which is now allocated and 418 * accounted for as ready for normal characters. This is used for drivers 419 * that need their own block copy routines into the buffer. There is no 420 * guarantee the buffer is a DMA target! 421 */ 422 423 int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars, 424 size_t size) 425 { 426 int space = __tty_buffer_request_room(port, size, TTYB_NORMAL); 427 if (likely(space)) { 428 struct tty_buffer *tb = port->buf.tail; 429 *chars = char_buf_ptr(tb, tb->used); 430 if (~tb->flags & TTYB_NORMAL) 431 memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space); 432 tb->used += space; 433 } 434 return space; 435 } 436 EXPORT_SYMBOL_GPL(tty_prepare_flip_string); 437 438 /** 439 * tty_ldisc_receive_buf - forward data to line discipline 440 * @ld: line discipline to process input 441 * @p: char buffer 442 * @f: TTY_* flags buffer 443 * @count: number of bytes to process 444 * 445 * Callers other than flush_to_ldisc() need to exclude the kworker 446 * from concurrent use of the line discipline, see paste_selection(). 447 * 448 * Returns the number of bytes not processed 449 */ 450 int tty_ldisc_receive_buf(struct tty_ldisc *ld, unsigned char *p, 451 char *f, int count) 452 { 453 if (ld->ops->receive_buf2) 454 count = ld->ops->receive_buf2(ld->tty, p, f, count); 455 else { 456 count = min_t(int, count, ld->tty->receive_room); 457 if (count && ld->ops->receive_buf) 458 ld->ops->receive_buf(ld->tty, p, f, count); 459 } 460 return count; 461 } 462 EXPORT_SYMBOL_GPL(tty_ldisc_receive_buf); 463 464 static int 465 receive_buf(struct tty_ldisc *ld, struct tty_buffer *head, int count) 466 { 467 unsigned char *p = char_buf_ptr(head, head->read); 468 char *f = NULL; 469 470 if (~head->flags & TTYB_NORMAL) 471 f = flag_buf_ptr(head, head->read); 472 473 return tty_ldisc_receive_buf(ld, p, f, count); 474 } 475 476 /** 477 * flush_to_ldisc 478 * @work: tty structure passed from work queue. 479 * 480 * This routine is called out of the software interrupt to flush data 481 * from the buffer chain to the line discipline. 482 * 483 * The receive_buf method is single threaded for each tty instance. 484 * 485 * Locking: takes buffer lock to ensure single-threaded flip buffer 486 * 'consumer' 487 */ 488 489 static void flush_to_ldisc(struct work_struct *work) 490 { 491 struct tty_port *port = container_of(work, struct tty_port, buf.work); 492 struct tty_bufhead *buf = &port->buf; 493 struct tty_struct *tty; 494 struct tty_ldisc *disc; 495 496 tty = READ_ONCE(port->itty); 497 if (tty == NULL) 498 return; 499 500 disc = tty_ldisc_ref(tty); 501 if (disc == NULL) 502 return; 503 504 mutex_lock(&buf->lock); 505 506 while (1) { 507 struct tty_buffer *head = buf->head; 508 struct tty_buffer *next; 509 int count; 510 511 /* Ldisc or user is trying to gain exclusive access */ 512 if (atomic_read(&buf->priority)) 513 break; 514 515 /* paired w/ release in __tty_buffer_request_room(); 516 * ensures commit value read is not stale if the head 517 * is advancing to the next buffer 518 */ 519 next = smp_load_acquire(&head->next); 520 /* paired w/ release in __tty_buffer_request_room() or in 521 * tty_buffer_flush(); ensures we see the committed buffer data 522 */ 523 count = smp_load_acquire(&head->commit) - head->read; 524 if (!count) { 525 if (next == NULL) { 526 check_other_closed(tty); 527 break; 528 } 529 buf->head = next; 530 tty_buffer_free(port, head); 531 continue; 532 } 533 534 count = receive_buf(disc, head, count); 535 if (!count) 536 break; 537 head->read += count; 538 } 539 540 mutex_unlock(&buf->lock); 541 542 tty_ldisc_deref(disc); 543 } 544 545 /** 546 * tty_flip_buffer_push - terminal 547 * @port: tty port to push 548 * 549 * Queue a push of the terminal flip buffers to the line discipline. 550 * Can be called from IRQ/atomic context. 551 * 552 * In the event of the queue being busy for flipping the work will be 553 * held off and retried later. 554 */ 555 556 void tty_flip_buffer_push(struct tty_port *port) 557 { 558 tty_schedule_flip(port); 559 } 560 EXPORT_SYMBOL(tty_flip_buffer_push); 561 562 /** 563 * tty_buffer_init - prepare a tty buffer structure 564 * @tty: tty to initialise 565 * 566 * Set up the initial state of the buffer management for a tty device. 567 * Must be called before the other tty buffer functions are used. 568 */ 569 570 void tty_buffer_init(struct tty_port *port) 571 { 572 struct tty_bufhead *buf = &port->buf; 573 574 mutex_init(&buf->lock); 575 tty_buffer_reset(&buf->sentinel, 0); 576 buf->head = &buf->sentinel; 577 buf->tail = &buf->sentinel; 578 init_llist_head(&buf->free); 579 atomic_set(&buf->mem_used, 0); 580 atomic_set(&buf->priority, 0); 581 INIT_WORK(&buf->work, flush_to_ldisc); 582 buf->mem_limit = TTYB_DEFAULT_MEM_LIMIT; 583 } 584 585 /** 586 * tty_buffer_set_limit - change the tty buffer memory limit 587 * @port: tty port to change 588 * 589 * Change the tty buffer memory limit. 590 * Must be called before the other tty buffer functions are used. 591 */ 592 593 int tty_buffer_set_limit(struct tty_port *port, int limit) 594 { 595 if (limit < MIN_TTYB_SIZE) 596 return -EINVAL; 597 port->buf.mem_limit = limit; 598 return 0; 599 } 600 EXPORT_SYMBOL_GPL(tty_buffer_set_limit); 601 602 /* slave ptys can claim nested buffer lock when handling BRK and INTR */ 603 void tty_buffer_set_lock_subclass(struct tty_port *port) 604 { 605 lockdep_set_subclass(&port->buf.lock, TTY_LOCK_SLAVE); 606 } 607 608 bool tty_buffer_restart_work(struct tty_port *port) 609 { 610 return queue_work(system_unbound_wq, &port->buf.work); 611 } 612 613 bool tty_buffer_cancel_work(struct tty_port *port) 614 { 615 return cancel_work_sync(&port->buf.work); 616 } 617