1 /* binder_alloc.c 2 * 3 * Android IPC Subsystem 4 * 5 * Copyright (C) 2007-2017 Google, Inc. 6 * 7 * This software is licensed under the terms of the GNU General Public 8 * License version 2, as published by the Free Software Foundation, and 9 * may be copied, distributed, and modified under those terms. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 */ 17 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <asm/cacheflush.h> 21 #include <linux/list.h> 22 #include <linux/sched/mm.h> 23 #include <linux/module.h> 24 #include <linux/rtmutex.h> 25 #include <linux/rbtree.h> 26 #include <linux/seq_file.h> 27 #include <linux/vmalloc.h> 28 #include <linux/slab.h> 29 #include <linux/sched.h> 30 #include <linux/list_lru.h> 31 #include "binder_alloc.h" 32 #include "binder_trace.h" 33 34 struct list_lru binder_alloc_lru; 35 36 static DEFINE_MUTEX(binder_alloc_mmap_lock); 37 38 enum { 39 BINDER_DEBUG_OPEN_CLOSE = 1U << 1, 40 BINDER_DEBUG_BUFFER_ALLOC = 1U << 2, 41 BINDER_DEBUG_BUFFER_ALLOC_ASYNC = 1U << 3, 42 }; 43 static uint32_t binder_alloc_debug_mask; 44 45 module_param_named(debug_mask, binder_alloc_debug_mask, 46 uint, 0644); 47 48 #define binder_alloc_debug(mask, x...) \ 49 do { \ 50 if (binder_alloc_debug_mask & mask) \ 51 pr_info(x); \ 52 } while (0) 53 54 static struct binder_buffer *binder_buffer_next(struct binder_buffer *buffer) 55 { 56 return list_entry(buffer->entry.next, struct binder_buffer, entry); 57 } 58 59 static struct binder_buffer *binder_buffer_prev(struct binder_buffer *buffer) 60 { 61 return list_entry(buffer->entry.prev, struct binder_buffer, entry); 62 } 63 64 static size_t binder_alloc_buffer_size(struct binder_alloc *alloc, 65 struct binder_buffer *buffer) 66 { 67 if (list_is_last(&buffer->entry, &alloc->buffers)) 68 return (u8 *)alloc->buffer + 69 alloc->buffer_size - (u8 *)buffer->data; 70 return (u8 *)binder_buffer_next(buffer)->data - (u8 *)buffer->data; 71 } 72 73 static void binder_insert_free_buffer(struct binder_alloc *alloc, 74 struct binder_buffer *new_buffer) 75 { 76 struct rb_node **p = &alloc->free_buffers.rb_node; 77 struct rb_node *parent = NULL; 78 struct binder_buffer *buffer; 79 size_t buffer_size; 80 size_t new_buffer_size; 81 82 BUG_ON(!new_buffer->free); 83 84 new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer); 85 86 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 87 "%d: add free buffer, size %zd, at %pK\n", 88 alloc->pid, new_buffer_size, new_buffer); 89 90 while (*p) { 91 parent = *p; 92 buffer = rb_entry(parent, struct binder_buffer, rb_node); 93 BUG_ON(!buffer->free); 94 95 buffer_size = binder_alloc_buffer_size(alloc, buffer); 96 97 if (new_buffer_size < buffer_size) 98 p = &parent->rb_left; 99 else 100 p = &parent->rb_right; 101 } 102 rb_link_node(&new_buffer->rb_node, parent, p); 103 rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers); 104 } 105 106 static void binder_insert_allocated_buffer_locked( 107 struct binder_alloc *alloc, struct binder_buffer *new_buffer) 108 { 109 struct rb_node **p = &alloc->allocated_buffers.rb_node; 110 struct rb_node *parent = NULL; 111 struct binder_buffer *buffer; 112 113 BUG_ON(new_buffer->free); 114 115 while (*p) { 116 parent = *p; 117 buffer = rb_entry(parent, struct binder_buffer, rb_node); 118 BUG_ON(buffer->free); 119 120 if (new_buffer->data < buffer->data) 121 p = &parent->rb_left; 122 else if (new_buffer->data > buffer->data) 123 p = &parent->rb_right; 124 else 125 BUG(); 126 } 127 rb_link_node(&new_buffer->rb_node, parent, p); 128 rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers); 129 } 130 131 static struct binder_buffer *binder_alloc_prepare_to_free_locked( 132 struct binder_alloc *alloc, 133 uintptr_t user_ptr) 134 { 135 struct rb_node *n = alloc->allocated_buffers.rb_node; 136 struct binder_buffer *buffer; 137 void *kern_ptr; 138 139 kern_ptr = (void *)(user_ptr - alloc->user_buffer_offset); 140 141 while (n) { 142 buffer = rb_entry(n, struct binder_buffer, rb_node); 143 BUG_ON(buffer->free); 144 145 if (kern_ptr < buffer->data) 146 n = n->rb_left; 147 else if (kern_ptr > buffer->data) 148 n = n->rb_right; 149 else { 150 /* 151 * Guard against user threads attempting to 152 * free the buffer twice 153 */ 154 if (buffer->free_in_progress) { 155 pr_err("%d:%d FREE_BUFFER u%016llx user freed buffer twice\n", 156 alloc->pid, current->pid, (u64)user_ptr); 157 return NULL; 158 } 159 buffer->free_in_progress = 1; 160 return buffer; 161 } 162 } 163 return NULL; 164 } 165 166 /** 167 * binder_alloc_buffer_lookup() - get buffer given user ptr 168 * @alloc: binder_alloc for this proc 169 * @user_ptr: User pointer to buffer data 170 * 171 * Validate userspace pointer to buffer data and return buffer corresponding to 172 * that user pointer. Search the rb tree for buffer that matches user data 173 * pointer. 174 * 175 * Return: Pointer to buffer or NULL 176 */ 177 struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc, 178 uintptr_t user_ptr) 179 { 180 struct binder_buffer *buffer; 181 182 mutex_lock(&alloc->mutex); 183 buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr); 184 mutex_unlock(&alloc->mutex); 185 return buffer; 186 } 187 188 static int binder_update_page_range(struct binder_alloc *alloc, int allocate, 189 void *start, void *end) 190 { 191 void *page_addr; 192 unsigned long user_page_addr; 193 struct binder_lru_page *page; 194 struct vm_area_struct *vma = NULL; 195 struct mm_struct *mm = NULL; 196 bool need_mm = false; 197 198 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 199 "%d: %s pages %pK-%pK\n", alloc->pid, 200 allocate ? "allocate" : "free", start, end); 201 202 if (end <= start) 203 return 0; 204 205 trace_binder_update_page_range(alloc, allocate, start, end); 206 207 if (allocate == 0) 208 goto free_range; 209 210 for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { 211 page = &alloc->pages[(page_addr - alloc->buffer) / PAGE_SIZE]; 212 if (!page->page_ptr) { 213 need_mm = true; 214 break; 215 } 216 } 217 218 if (need_mm && mmget_not_zero(alloc->vma_vm_mm)) 219 mm = alloc->vma_vm_mm; 220 221 if (mm) { 222 down_write(&mm->mmap_sem); 223 vma = alloc->vma; 224 } 225 226 if (!vma && need_mm) { 227 pr_err("%d: binder_alloc_buf failed to map pages in userspace, no vma\n", 228 alloc->pid); 229 goto err_no_vma; 230 } 231 232 for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { 233 int ret; 234 bool on_lru; 235 size_t index; 236 237 index = (page_addr - alloc->buffer) / PAGE_SIZE; 238 page = &alloc->pages[index]; 239 240 if (page->page_ptr) { 241 trace_binder_alloc_lru_start(alloc, index); 242 243 on_lru = list_lru_del(&binder_alloc_lru, &page->lru); 244 WARN_ON(!on_lru); 245 246 trace_binder_alloc_lru_end(alloc, index); 247 continue; 248 } 249 250 if (WARN_ON(!vma)) 251 goto err_page_ptr_cleared; 252 253 trace_binder_alloc_page_start(alloc, index); 254 page->page_ptr = alloc_page(GFP_KERNEL | 255 __GFP_HIGHMEM | 256 __GFP_ZERO); 257 if (!page->page_ptr) { 258 pr_err("%d: binder_alloc_buf failed for page at %pK\n", 259 alloc->pid, page_addr); 260 goto err_alloc_page_failed; 261 } 262 page->alloc = alloc; 263 INIT_LIST_HEAD(&page->lru); 264 265 ret = map_kernel_range_noflush((unsigned long)page_addr, 266 PAGE_SIZE, PAGE_KERNEL, 267 &page->page_ptr); 268 flush_cache_vmap((unsigned long)page_addr, 269 (unsigned long)page_addr + PAGE_SIZE); 270 if (ret != 1) { 271 pr_err("%d: binder_alloc_buf failed to map page at %pK in kernel\n", 272 alloc->pid, page_addr); 273 goto err_map_kernel_failed; 274 } 275 user_page_addr = 276 (uintptr_t)page_addr + alloc->user_buffer_offset; 277 ret = vm_insert_page(vma, user_page_addr, page[0].page_ptr); 278 if (ret) { 279 pr_err("%d: binder_alloc_buf failed to map page at %lx in userspace\n", 280 alloc->pid, user_page_addr); 281 goto err_vm_insert_page_failed; 282 } 283 284 if (index + 1 > alloc->pages_high) 285 alloc->pages_high = index + 1; 286 287 trace_binder_alloc_page_end(alloc, index); 288 /* vm_insert_page does not seem to increment the refcount */ 289 } 290 if (mm) { 291 up_write(&mm->mmap_sem); 292 mmput(mm); 293 } 294 return 0; 295 296 free_range: 297 for (page_addr = end - PAGE_SIZE; page_addr >= start; 298 page_addr -= PAGE_SIZE) { 299 bool ret; 300 size_t index; 301 302 index = (page_addr - alloc->buffer) / PAGE_SIZE; 303 page = &alloc->pages[index]; 304 305 trace_binder_free_lru_start(alloc, index); 306 307 ret = list_lru_add(&binder_alloc_lru, &page->lru); 308 WARN_ON(!ret); 309 310 trace_binder_free_lru_end(alloc, index); 311 continue; 312 313 err_vm_insert_page_failed: 314 unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE); 315 err_map_kernel_failed: 316 __free_page(page->page_ptr); 317 page->page_ptr = NULL; 318 err_alloc_page_failed: 319 err_page_ptr_cleared: 320 ; 321 } 322 err_no_vma: 323 if (mm) { 324 up_write(&mm->mmap_sem); 325 mmput(mm); 326 } 327 return vma ? -ENOMEM : -ESRCH; 328 } 329 330 struct binder_buffer *binder_alloc_new_buf_locked(struct binder_alloc *alloc, 331 size_t data_size, 332 size_t offsets_size, 333 size_t extra_buffers_size, 334 int is_async) 335 { 336 struct rb_node *n = alloc->free_buffers.rb_node; 337 struct binder_buffer *buffer; 338 size_t buffer_size; 339 struct rb_node *best_fit = NULL; 340 void *has_page_addr; 341 void *end_page_addr; 342 size_t size, data_offsets_size; 343 int ret; 344 345 if (alloc->vma == NULL) { 346 pr_err("%d: binder_alloc_buf, no vma\n", 347 alloc->pid); 348 return ERR_PTR(-ESRCH); 349 } 350 351 data_offsets_size = ALIGN(data_size, sizeof(void *)) + 352 ALIGN(offsets_size, sizeof(void *)); 353 354 if (data_offsets_size < data_size || data_offsets_size < offsets_size) { 355 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 356 "%d: got transaction with invalid size %zd-%zd\n", 357 alloc->pid, data_size, offsets_size); 358 return ERR_PTR(-EINVAL); 359 } 360 size = data_offsets_size + ALIGN(extra_buffers_size, sizeof(void *)); 361 if (size < data_offsets_size || size < extra_buffers_size) { 362 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 363 "%d: got transaction with invalid extra_buffers_size %zd\n", 364 alloc->pid, extra_buffers_size); 365 return ERR_PTR(-EINVAL); 366 } 367 if (is_async && 368 alloc->free_async_space < size + sizeof(struct binder_buffer)) { 369 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 370 "%d: binder_alloc_buf size %zd failed, no async space left\n", 371 alloc->pid, size); 372 return ERR_PTR(-ENOSPC); 373 } 374 375 /* Pad 0-size buffers so they get assigned unique addresses */ 376 size = max(size, sizeof(void *)); 377 378 while (n) { 379 buffer = rb_entry(n, struct binder_buffer, rb_node); 380 BUG_ON(!buffer->free); 381 buffer_size = binder_alloc_buffer_size(alloc, buffer); 382 383 if (size < buffer_size) { 384 best_fit = n; 385 n = n->rb_left; 386 } else if (size > buffer_size) 387 n = n->rb_right; 388 else { 389 best_fit = n; 390 break; 391 } 392 } 393 if (best_fit == NULL) { 394 size_t allocated_buffers = 0; 395 size_t largest_alloc_size = 0; 396 size_t total_alloc_size = 0; 397 size_t free_buffers = 0; 398 size_t largest_free_size = 0; 399 size_t total_free_size = 0; 400 401 for (n = rb_first(&alloc->allocated_buffers); n != NULL; 402 n = rb_next(n)) { 403 buffer = rb_entry(n, struct binder_buffer, rb_node); 404 buffer_size = binder_alloc_buffer_size(alloc, buffer); 405 allocated_buffers++; 406 total_alloc_size += buffer_size; 407 if (buffer_size > largest_alloc_size) 408 largest_alloc_size = buffer_size; 409 } 410 for (n = rb_first(&alloc->free_buffers); n != NULL; 411 n = rb_next(n)) { 412 buffer = rb_entry(n, struct binder_buffer, rb_node); 413 buffer_size = binder_alloc_buffer_size(alloc, buffer); 414 free_buffers++; 415 total_free_size += buffer_size; 416 if (buffer_size > largest_free_size) 417 largest_free_size = buffer_size; 418 } 419 pr_err("%d: binder_alloc_buf size %zd failed, no address space\n", 420 alloc->pid, size); 421 pr_err("allocated: %zd (num: %zd largest: %zd), free: %zd (num: %zd largest: %zd)\n", 422 total_alloc_size, allocated_buffers, largest_alloc_size, 423 total_free_size, free_buffers, largest_free_size); 424 return ERR_PTR(-ENOSPC); 425 } 426 if (n == NULL) { 427 buffer = rb_entry(best_fit, struct binder_buffer, rb_node); 428 buffer_size = binder_alloc_buffer_size(alloc, buffer); 429 } 430 431 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 432 "%d: binder_alloc_buf size %zd got buffer %pK size %zd\n", 433 alloc->pid, size, buffer, buffer_size); 434 435 has_page_addr = 436 (void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK); 437 WARN_ON(n && buffer_size != size); 438 end_page_addr = 439 (void *)PAGE_ALIGN((uintptr_t)buffer->data + size); 440 if (end_page_addr > has_page_addr) 441 end_page_addr = has_page_addr; 442 ret = binder_update_page_range(alloc, 1, 443 (void *)PAGE_ALIGN((uintptr_t)buffer->data), end_page_addr); 444 if (ret) 445 return ERR_PTR(ret); 446 447 if (buffer_size != size) { 448 struct binder_buffer *new_buffer; 449 450 new_buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); 451 if (!new_buffer) { 452 pr_err("%s: %d failed to alloc new buffer struct\n", 453 __func__, alloc->pid); 454 goto err_alloc_buf_struct_failed; 455 } 456 new_buffer->data = (u8 *)buffer->data + size; 457 list_add(&new_buffer->entry, &buffer->entry); 458 new_buffer->free = 1; 459 binder_insert_free_buffer(alloc, new_buffer); 460 } 461 462 rb_erase(best_fit, &alloc->free_buffers); 463 buffer->free = 0; 464 buffer->free_in_progress = 0; 465 binder_insert_allocated_buffer_locked(alloc, buffer); 466 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 467 "%d: binder_alloc_buf size %zd got %pK\n", 468 alloc->pid, size, buffer); 469 buffer->data_size = data_size; 470 buffer->offsets_size = offsets_size; 471 buffer->async_transaction = is_async; 472 buffer->extra_buffers_size = extra_buffers_size; 473 if (is_async) { 474 alloc->free_async_space -= size + sizeof(struct binder_buffer); 475 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, 476 "%d: binder_alloc_buf size %zd async free %zd\n", 477 alloc->pid, size, alloc->free_async_space); 478 } 479 return buffer; 480 481 err_alloc_buf_struct_failed: 482 binder_update_page_range(alloc, 0, 483 (void *)PAGE_ALIGN((uintptr_t)buffer->data), 484 end_page_addr); 485 return ERR_PTR(-ENOMEM); 486 } 487 488 /** 489 * binder_alloc_new_buf() - Allocate a new binder buffer 490 * @alloc: binder_alloc for this proc 491 * @data_size: size of user data buffer 492 * @offsets_size: user specified buffer offset 493 * @extra_buffers_size: size of extra space for meta-data (eg, security context) 494 * @is_async: buffer for async transaction 495 * 496 * Allocate a new buffer given the requested sizes. Returns 497 * the kernel version of the buffer pointer. The size allocated 498 * is the sum of the three given sizes (each rounded up to 499 * pointer-sized boundary) 500 * 501 * Return: The allocated buffer or %NULL if error 502 */ 503 struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc, 504 size_t data_size, 505 size_t offsets_size, 506 size_t extra_buffers_size, 507 int is_async) 508 { 509 struct binder_buffer *buffer; 510 511 mutex_lock(&alloc->mutex); 512 buffer = binder_alloc_new_buf_locked(alloc, data_size, offsets_size, 513 extra_buffers_size, is_async); 514 mutex_unlock(&alloc->mutex); 515 return buffer; 516 } 517 518 static void *buffer_start_page(struct binder_buffer *buffer) 519 { 520 return (void *)((uintptr_t)buffer->data & PAGE_MASK); 521 } 522 523 static void *prev_buffer_end_page(struct binder_buffer *buffer) 524 { 525 return (void *)(((uintptr_t)(buffer->data) - 1) & PAGE_MASK); 526 } 527 528 static void binder_delete_free_buffer(struct binder_alloc *alloc, 529 struct binder_buffer *buffer) 530 { 531 struct binder_buffer *prev, *next = NULL; 532 bool to_free = true; 533 BUG_ON(alloc->buffers.next == &buffer->entry); 534 prev = binder_buffer_prev(buffer); 535 BUG_ON(!prev->free); 536 if (prev_buffer_end_page(prev) == buffer_start_page(buffer)) { 537 to_free = false; 538 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 539 "%d: merge free, buffer %pK share page with %pK\n", 540 alloc->pid, buffer->data, prev->data); 541 } 542 543 if (!list_is_last(&buffer->entry, &alloc->buffers)) { 544 next = binder_buffer_next(buffer); 545 if (buffer_start_page(next) == buffer_start_page(buffer)) { 546 to_free = false; 547 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 548 "%d: merge free, buffer %pK share page with %pK\n", 549 alloc->pid, 550 buffer->data, 551 next->data); 552 } 553 } 554 555 if (PAGE_ALIGNED(buffer->data)) { 556 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 557 "%d: merge free, buffer start %pK is page aligned\n", 558 alloc->pid, buffer->data); 559 to_free = false; 560 } 561 562 if (to_free) { 563 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 564 "%d: merge free, buffer %pK do not share page with %pK or %pK\n", 565 alloc->pid, buffer->data, 566 prev->data, next ? next->data : NULL); 567 binder_update_page_range(alloc, 0, buffer_start_page(buffer), 568 buffer_start_page(buffer) + PAGE_SIZE); 569 } 570 list_del(&buffer->entry); 571 kfree(buffer); 572 } 573 574 static void binder_free_buf_locked(struct binder_alloc *alloc, 575 struct binder_buffer *buffer) 576 { 577 size_t size, buffer_size; 578 579 buffer_size = binder_alloc_buffer_size(alloc, buffer); 580 581 size = ALIGN(buffer->data_size, sizeof(void *)) + 582 ALIGN(buffer->offsets_size, sizeof(void *)) + 583 ALIGN(buffer->extra_buffers_size, sizeof(void *)); 584 585 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 586 "%d: binder_free_buf %pK size %zd buffer_size %zd\n", 587 alloc->pid, buffer, size, buffer_size); 588 589 BUG_ON(buffer->free); 590 BUG_ON(size > buffer_size); 591 BUG_ON(buffer->transaction != NULL); 592 BUG_ON(buffer->data < alloc->buffer); 593 BUG_ON(buffer->data > alloc->buffer + alloc->buffer_size); 594 595 if (buffer->async_transaction) { 596 alloc->free_async_space += size + sizeof(struct binder_buffer); 597 598 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, 599 "%d: binder_free_buf size %zd async free %zd\n", 600 alloc->pid, size, alloc->free_async_space); 601 } 602 603 binder_update_page_range(alloc, 0, 604 (void *)PAGE_ALIGN((uintptr_t)buffer->data), 605 (void *)(((uintptr_t)buffer->data + buffer_size) & PAGE_MASK)); 606 607 rb_erase(&buffer->rb_node, &alloc->allocated_buffers); 608 buffer->free = 1; 609 if (!list_is_last(&buffer->entry, &alloc->buffers)) { 610 struct binder_buffer *next = binder_buffer_next(buffer); 611 612 if (next->free) { 613 rb_erase(&next->rb_node, &alloc->free_buffers); 614 binder_delete_free_buffer(alloc, next); 615 } 616 } 617 if (alloc->buffers.next != &buffer->entry) { 618 struct binder_buffer *prev = binder_buffer_prev(buffer); 619 620 if (prev->free) { 621 binder_delete_free_buffer(alloc, buffer); 622 rb_erase(&prev->rb_node, &alloc->free_buffers); 623 buffer = prev; 624 } 625 } 626 binder_insert_free_buffer(alloc, buffer); 627 } 628 629 /** 630 * binder_alloc_free_buf() - free a binder buffer 631 * @alloc: binder_alloc for this proc 632 * @buffer: kernel pointer to buffer 633 * 634 * Free the buffer allocated via binder_alloc_new_buffer() 635 */ 636 void binder_alloc_free_buf(struct binder_alloc *alloc, 637 struct binder_buffer *buffer) 638 { 639 mutex_lock(&alloc->mutex); 640 binder_free_buf_locked(alloc, buffer); 641 mutex_unlock(&alloc->mutex); 642 } 643 644 /** 645 * binder_alloc_mmap_handler() - map virtual address space for proc 646 * @alloc: alloc structure for this proc 647 * @vma: vma passed to mmap() 648 * 649 * Called by binder_mmap() to initialize the space specified in 650 * vma for allocating binder buffers 651 * 652 * Return: 653 * 0 = success 654 * -EBUSY = address space already mapped 655 * -ENOMEM = failed to map memory to given address space 656 */ 657 int binder_alloc_mmap_handler(struct binder_alloc *alloc, 658 struct vm_area_struct *vma) 659 { 660 int ret; 661 struct vm_struct *area; 662 const char *failure_string; 663 struct binder_buffer *buffer; 664 665 mutex_lock(&binder_alloc_mmap_lock); 666 if (alloc->buffer) { 667 ret = -EBUSY; 668 failure_string = "already mapped"; 669 goto err_already_mapped; 670 } 671 672 area = get_vm_area(vma->vm_end - vma->vm_start, VM_IOREMAP); 673 if (area == NULL) { 674 ret = -ENOMEM; 675 failure_string = "get_vm_area"; 676 goto err_get_vm_area_failed; 677 } 678 alloc->buffer = area->addr; 679 alloc->user_buffer_offset = 680 vma->vm_start - (uintptr_t)alloc->buffer; 681 mutex_unlock(&binder_alloc_mmap_lock); 682 683 #ifdef CONFIG_CPU_CACHE_VIPT 684 if (cache_is_vipt_aliasing()) { 685 while (CACHE_COLOUR( 686 (vma->vm_start ^ (uint32_t)alloc->buffer))) { 687 pr_info("%s: %d %lx-%lx maps %pK bad alignment\n", 688 __func__, alloc->pid, vma->vm_start, 689 vma->vm_end, alloc->buffer); 690 vma->vm_start += PAGE_SIZE; 691 } 692 } 693 #endif 694 alloc->pages = kzalloc(sizeof(alloc->pages[0]) * 695 ((vma->vm_end - vma->vm_start) / PAGE_SIZE), 696 GFP_KERNEL); 697 if (alloc->pages == NULL) { 698 ret = -ENOMEM; 699 failure_string = "alloc page array"; 700 goto err_alloc_pages_failed; 701 } 702 alloc->buffer_size = vma->vm_end - vma->vm_start; 703 704 buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); 705 if (!buffer) { 706 ret = -ENOMEM; 707 failure_string = "alloc buffer struct"; 708 goto err_alloc_buf_struct_failed; 709 } 710 711 buffer->data = alloc->buffer; 712 list_add(&buffer->entry, &alloc->buffers); 713 buffer->free = 1; 714 binder_insert_free_buffer(alloc, buffer); 715 alloc->free_async_space = alloc->buffer_size / 2; 716 barrier(); 717 alloc->vma = vma; 718 alloc->vma_vm_mm = vma->vm_mm; 719 mmgrab(alloc->vma_vm_mm); 720 721 return 0; 722 723 err_alloc_buf_struct_failed: 724 kfree(alloc->pages); 725 alloc->pages = NULL; 726 err_alloc_pages_failed: 727 mutex_lock(&binder_alloc_mmap_lock); 728 vfree(alloc->buffer); 729 alloc->buffer = NULL; 730 err_get_vm_area_failed: 731 err_already_mapped: 732 mutex_unlock(&binder_alloc_mmap_lock); 733 pr_err("%s: %d %lx-%lx %s failed %d\n", __func__, 734 alloc->pid, vma->vm_start, vma->vm_end, failure_string, ret); 735 return ret; 736 } 737 738 739 void binder_alloc_deferred_release(struct binder_alloc *alloc) 740 { 741 struct rb_node *n; 742 int buffers, page_count; 743 struct binder_buffer *buffer; 744 745 BUG_ON(alloc->vma); 746 747 buffers = 0; 748 mutex_lock(&alloc->mutex); 749 while ((n = rb_first(&alloc->allocated_buffers))) { 750 buffer = rb_entry(n, struct binder_buffer, rb_node); 751 752 /* Transaction should already have been freed */ 753 BUG_ON(buffer->transaction); 754 755 binder_free_buf_locked(alloc, buffer); 756 buffers++; 757 } 758 759 while (!list_empty(&alloc->buffers)) { 760 buffer = list_first_entry(&alloc->buffers, 761 struct binder_buffer, entry); 762 WARN_ON(!buffer->free); 763 764 list_del(&buffer->entry); 765 WARN_ON_ONCE(!list_empty(&alloc->buffers)); 766 kfree(buffer); 767 } 768 769 page_count = 0; 770 if (alloc->pages) { 771 int i; 772 773 for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { 774 void *page_addr; 775 bool on_lru; 776 777 if (!alloc->pages[i].page_ptr) 778 continue; 779 780 on_lru = list_lru_del(&binder_alloc_lru, 781 &alloc->pages[i].lru); 782 page_addr = alloc->buffer + i * PAGE_SIZE; 783 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, 784 "%s: %d: page %d at %pK %s\n", 785 __func__, alloc->pid, i, page_addr, 786 on_lru ? "on lru" : "active"); 787 unmap_kernel_range((unsigned long)page_addr, PAGE_SIZE); 788 __free_page(alloc->pages[i].page_ptr); 789 page_count++; 790 } 791 kfree(alloc->pages); 792 vfree(alloc->buffer); 793 } 794 mutex_unlock(&alloc->mutex); 795 if (alloc->vma_vm_mm) 796 mmdrop(alloc->vma_vm_mm); 797 798 binder_alloc_debug(BINDER_DEBUG_OPEN_CLOSE, 799 "%s: %d buffers %d, pages %d\n", 800 __func__, alloc->pid, buffers, page_count); 801 } 802 803 static void print_binder_buffer(struct seq_file *m, const char *prefix, 804 struct binder_buffer *buffer) 805 { 806 seq_printf(m, "%s %d: %pK size %zd:%zd:%zd %s\n", 807 prefix, buffer->debug_id, buffer->data, 808 buffer->data_size, buffer->offsets_size, 809 buffer->extra_buffers_size, 810 buffer->transaction ? "active" : "delivered"); 811 } 812 813 /** 814 * binder_alloc_print_allocated() - print buffer info 815 * @m: seq_file for output via seq_printf() 816 * @alloc: binder_alloc for this proc 817 * 818 * Prints information about every buffer associated with 819 * the binder_alloc state to the given seq_file 820 */ 821 void binder_alloc_print_allocated(struct seq_file *m, 822 struct binder_alloc *alloc) 823 { 824 struct rb_node *n; 825 826 mutex_lock(&alloc->mutex); 827 for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) 828 print_binder_buffer(m, " buffer", 829 rb_entry(n, struct binder_buffer, rb_node)); 830 mutex_unlock(&alloc->mutex); 831 } 832 833 /** 834 * binder_alloc_print_pages() - print page usage 835 * @m: seq_file for output via seq_printf() 836 * @alloc: binder_alloc for this proc 837 */ 838 void binder_alloc_print_pages(struct seq_file *m, 839 struct binder_alloc *alloc) 840 { 841 struct binder_lru_page *page; 842 int i; 843 int active = 0; 844 int lru = 0; 845 int free = 0; 846 847 mutex_lock(&alloc->mutex); 848 for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { 849 page = &alloc->pages[i]; 850 if (!page->page_ptr) 851 free++; 852 else if (list_empty(&page->lru)) 853 active++; 854 else 855 lru++; 856 } 857 mutex_unlock(&alloc->mutex); 858 seq_printf(m, " pages: %d:%d:%d\n", active, lru, free); 859 seq_printf(m, " pages high watermark: %zu\n", alloc->pages_high); 860 } 861 862 /** 863 * binder_alloc_get_allocated_count() - return count of buffers 864 * @alloc: binder_alloc for this proc 865 * 866 * Return: count of allocated buffers 867 */ 868 int binder_alloc_get_allocated_count(struct binder_alloc *alloc) 869 { 870 struct rb_node *n; 871 int count = 0; 872 873 mutex_lock(&alloc->mutex); 874 for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) 875 count++; 876 mutex_unlock(&alloc->mutex); 877 return count; 878 } 879 880 881 /** 882 * binder_alloc_vma_close() - invalidate address space 883 * @alloc: binder_alloc for this proc 884 * 885 * Called from binder_vma_close() when releasing address space. 886 * Clears alloc->vma to prevent new incoming transactions from 887 * allocating more buffers. 888 */ 889 void binder_alloc_vma_close(struct binder_alloc *alloc) 890 { 891 WRITE_ONCE(alloc->vma, NULL); 892 } 893 894 /** 895 * binder_alloc_free_page() - shrinker callback to free pages 896 * @item: item to free 897 * @lock: lock protecting the item 898 * @cb_arg: callback argument 899 * 900 * Called from list_lru_walk() in binder_shrink_scan() to free 901 * up pages when the system is under memory pressure. 902 */ 903 enum lru_status binder_alloc_free_page(struct list_head *item, 904 struct list_lru_one *lru, 905 spinlock_t *lock, 906 void *cb_arg) 907 { 908 struct mm_struct *mm = NULL; 909 struct binder_lru_page *page = container_of(item, 910 struct binder_lru_page, 911 lru); 912 struct binder_alloc *alloc; 913 uintptr_t page_addr; 914 size_t index; 915 struct vm_area_struct *vma; 916 917 alloc = page->alloc; 918 if (!mutex_trylock(&alloc->mutex)) 919 goto err_get_alloc_mutex_failed; 920 921 if (!page->page_ptr) 922 goto err_page_already_freed; 923 924 index = page - alloc->pages; 925 page_addr = (uintptr_t)alloc->buffer + index * PAGE_SIZE; 926 vma = alloc->vma; 927 if (vma) { 928 if (!mmget_not_zero(alloc->vma_vm_mm)) 929 goto err_mmget; 930 mm = alloc->vma_vm_mm; 931 if (!down_write_trylock(&mm->mmap_sem)) 932 goto err_down_write_mmap_sem_failed; 933 } 934 935 list_lru_isolate(lru, item); 936 spin_unlock(lock); 937 938 if (vma) { 939 trace_binder_unmap_user_start(alloc, index); 940 941 zap_page_range(vma, 942 page_addr + alloc->user_buffer_offset, 943 PAGE_SIZE); 944 945 trace_binder_unmap_user_end(alloc, index); 946 947 up_write(&mm->mmap_sem); 948 mmput(mm); 949 } 950 951 trace_binder_unmap_kernel_start(alloc, index); 952 953 unmap_kernel_range(page_addr, PAGE_SIZE); 954 __free_page(page->page_ptr); 955 page->page_ptr = NULL; 956 957 trace_binder_unmap_kernel_end(alloc, index); 958 959 spin_lock(lock); 960 mutex_unlock(&alloc->mutex); 961 return LRU_REMOVED_RETRY; 962 963 err_down_write_mmap_sem_failed: 964 mmput_async(mm); 965 err_mmget: 966 err_page_already_freed: 967 mutex_unlock(&alloc->mutex); 968 err_get_alloc_mutex_failed: 969 return LRU_SKIP; 970 } 971 972 static unsigned long 973 binder_shrink_count(struct shrinker *shrink, struct shrink_control *sc) 974 { 975 unsigned long ret = list_lru_count(&binder_alloc_lru); 976 return ret; 977 } 978 979 static unsigned long 980 binder_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) 981 { 982 unsigned long ret; 983 984 ret = list_lru_walk(&binder_alloc_lru, binder_alloc_free_page, 985 NULL, sc->nr_to_scan); 986 return ret; 987 } 988 989 static struct shrinker binder_shrinker = { 990 .count_objects = binder_shrink_count, 991 .scan_objects = binder_shrink_scan, 992 .seeks = DEFAULT_SEEKS, 993 }; 994 995 /** 996 * binder_alloc_init() - called by binder_open() for per-proc initialization 997 * @alloc: binder_alloc for this proc 998 * 999 * Called from binder_open() to initialize binder_alloc fields for 1000 * new binder proc 1001 */ 1002 void binder_alloc_init(struct binder_alloc *alloc) 1003 { 1004 alloc->pid = current->group_leader->pid; 1005 mutex_init(&alloc->mutex); 1006 INIT_LIST_HEAD(&alloc->buffers); 1007 } 1008 1009 int binder_alloc_shrinker_init(void) 1010 { 1011 int ret = list_lru_init(&binder_alloc_lru); 1012 1013 if (ret == 0) { 1014 ret = register_shrinker(&binder_shrinker); 1015 if (ret) 1016 list_lru_destroy(&binder_alloc_lru); 1017 } 1018 return ret; 1019 } 1020