1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved. 4 * Copyright (c) 2010-2012 Broadcom. All rights reserved. 5 */ 6 7 #include <linux/cdev.h> 8 #include <linux/fs.h> 9 #include <linux/device.h> 10 #include <linux/slab.h> 11 #include <linux/compat.h> 12 #include <linux/miscdevice.h> 13 14 #include "vchiq_core.h" 15 #include "vchiq_ioctl.h" 16 #include "vchiq_arm.h" 17 #include "vchiq_debugfs.h" 18 19 static const char *const ioctl_names[] = { 20 "CONNECT", 21 "SHUTDOWN", 22 "CREATE_SERVICE", 23 "REMOVE_SERVICE", 24 "QUEUE_MESSAGE", 25 "QUEUE_BULK_TRANSMIT", 26 "QUEUE_BULK_RECEIVE", 27 "AWAIT_COMPLETION", 28 "DEQUEUE_MESSAGE", 29 "GET_CLIENT_ID", 30 "GET_CONFIG", 31 "CLOSE_SERVICE", 32 "USE_SERVICE", 33 "RELEASE_SERVICE", 34 "SET_SERVICE_OPTION", 35 "DUMP_PHYS_MEM", 36 "LIB_VERSION", 37 "CLOSE_DELIVERED" 38 }; 39 40 static_assert(ARRAY_SIZE(ioctl_names) == (VCHIQ_IOC_MAX + 1)); 41 42 static void 43 user_service_free(void *userdata) 44 { 45 kfree(userdata); 46 } 47 48 static void close_delivered(struct user_service *user_service) 49 { 50 vchiq_log_info(vchiq_arm_log_level, 51 "%s(handle=%x)", 52 __func__, user_service->service->handle); 53 54 if (user_service->close_pending) { 55 /* Allow the underlying service to be culled */ 56 vchiq_service_put(user_service->service); 57 58 /* Wake the user-thread blocked in close_ or remove_service */ 59 complete(&user_service->close_event); 60 61 user_service->close_pending = 0; 62 } 63 } 64 65 struct vchiq_io_copy_callback_context { 66 struct vchiq_element *element; 67 size_t element_offset; 68 unsigned long elements_to_go; 69 }; 70 71 static ssize_t vchiq_ioc_copy_element_data(void *context, void *dest, 72 size_t offset, size_t maxsize) 73 { 74 struct vchiq_io_copy_callback_context *cc = context; 75 size_t total_bytes_copied = 0; 76 size_t bytes_this_round; 77 78 while (total_bytes_copied < maxsize) { 79 if (!cc->elements_to_go) 80 return total_bytes_copied; 81 82 if (!cc->element->size) { 83 cc->elements_to_go--; 84 cc->element++; 85 cc->element_offset = 0; 86 continue; 87 } 88 89 bytes_this_round = min(cc->element->size - cc->element_offset, 90 maxsize - total_bytes_copied); 91 92 if (copy_from_user(dest + total_bytes_copied, 93 cc->element->data + cc->element_offset, 94 bytes_this_round)) 95 return -EFAULT; 96 97 cc->element_offset += bytes_this_round; 98 total_bytes_copied += bytes_this_round; 99 100 if (cc->element_offset == cc->element->size) { 101 cc->elements_to_go--; 102 cc->element++; 103 cc->element_offset = 0; 104 } 105 } 106 107 return maxsize; 108 } 109 110 static int 111 vchiq_ioc_queue_message(unsigned int handle, struct vchiq_element *elements, 112 unsigned long count) 113 { 114 struct vchiq_io_copy_callback_context context; 115 enum vchiq_status status = VCHIQ_SUCCESS; 116 unsigned long i; 117 size_t total_size = 0; 118 119 context.element = elements; 120 context.element_offset = 0; 121 context.elements_to_go = count; 122 123 for (i = 0; i < count; i++) { 124 if (!elements[i].data && elements[i].size != 0) 125 return -EFAULT; 126 127 total_size += elements[i].size; 128 } 129 130 status = vchiq_queue_message(handle, vchiq_ioc_copy_element_data, 131 &context, total_size); 132 133 if (status == VCHIQ_ERROR) 134 return -EIO; 135 else if (status == VCHIQ_RETRY) 136 return -EINTR; 137 return 0; 138 } 139 140 static int vchiq_ioc_create_service(struct vchiq_instance *instance, 141 struct vchiq_create_service *args) 142 { 143 struct user_service *user_service = NULL; 144 struct vchiq_service *service; 145 enum vchiq_status status = VCHIQ_SUCCESS; 146 struct vchiq_service_params_kernel params; 147 int srvstate; 148 149 user_service = kmalloc(sizeof(*user_service), GFP_KERNEL); 150 if (!user_service) 151 return -ENOMEM; 152 153 if (args->is_open) { 154 if (!instance->connected) { 155 kfree(user_service); 156 return -ENOTCONN; 157 } 158 srvstate = VCHIQ_SRVSTATE_OPENING; 159 } else { 160 srvstate = instance->connected ? 161 VCHIQ_SRVSTATE_LISTENING : VCHIQ_SRVSTATE_HIDDEN; 162 } 163 164 params = (struct vchiq_service_params_kernel) { 165 .fourcc = args->params.fourcc, 166 .callback = service_callback, 167 .userdata = user_service, 168 .version = args->params.version, 169 .version_min = args->params.version_min, 170 }; 171 service = vchiq_add_service_internal(instance->state, ¶ms, 172 srvstate, instance, 173 user_service_free); 174 if (!service) { 175 kfree(user_service); 176 return -EEXIST; 177 } 178 179 user_service->service = service; 180 user_service->userdata = args->params.userdata; 181 user_service->instance = instance; 182 user_service->is_vchi = (args->is_vchi != 0); 183 user_service->dequeue_pending = 0; 184 user_service->close_pending = 0; 185 user_service->message_available_pos = instance->completion_remove - 1; 186 user_service->msg_insert = 0; 187 user_service->msg_remove = 0; 188 init_completion(&user_service->insert_event); 189 init_completion(&user_service->remove_event); 190 init_completion(&user_service->close_event); 191 192 if (args->is_open) { 193 status = vchiq_open_service_internal(service, instance->pid); 194 if (status != VCHIQ_SUCCESS) { 195 vchiq_remove_service(service->handle); 196 return (status == VCHIQ_RETRY) ? 197 -EINTR : -EIO; 198 } 199 } 200 args->handle = service->handle; 201 202 return 0; 203 } 204 205 static int vchiq_ioc_dequeue_message(struct vchiq_instance *instance, 206 struct vchiq_dequeue_message *args) 207 { 208 struct user_service *user_service; 209 struct vchiq_service *service; 210 struct vchiq_header *header; 211 int ret; 212 213 DEBUG_INITIALISE(g_state.local); 214 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE); 215 service = find_service_for_instance(instance, args->handle); 216 if (!service) 217 return -EINVAL; 218 219 user_service = (struct user_service *)service->base.userdata; 220 if (user_service->is_vchi == 0) { 221 ret = -EINVAL; 222 goto out; 223 } 224 225 spin_lock(&msg_queue_spinlock); 226 if (user_service->msg_remove == user_service->msg_insert) { 227 if (!args->blocking) { 228 spin_unlock(&msg_queue_spinlock); 229 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE); 230 ret = -EWOULDBLOCK; 231 goto out; 232 } 233 user_service->dequeue_pending = 1; 234 ret = 0; 235 do { 236 spin_unlock(&msg_queue_spinlock); 237 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE); 238 if (wait_for_completion_interruptible(&user_service->insert_event)) { 239 vchiq_log_info(vchiq_arm_log_level, 240 "DEQUEUE_MESSAGE interrupted"); 241 ret = -EINTR; 242 break; 243 } 244 spin_lock(&msg_queue_spinlock); 245 } while (user_service->msg_remove == user_service->msg_insert); 246 247 if (ret) 248 goto out; 249 } 250 251 if (WARN_ON_ONCE((int)(user_service->msg_insert - 252 user_service->msg_remove) < 0)) { 253 spin_unlock(&msg_queue_spinlock); 254 ret = -EINVAL; 255 goto out; 256 } 257 258 header = user_service->msg_queue[user_service->msg_remove & 259 (MSG_QUEUE_SIZE - 1)]; 260 user_service->msg_remove++; 261 spin_unlock(&msg_queue_spinlock); 262 263 complete(&user_service->remove_event); 264 if (!header) { 265 ret = -ENOTCONN; 266 } else if (header->size <= args->bufsize) { 267 /* Copy to user space if msgbuf is not NULL */ 268 if (!args->buf || (copy_to_user(args->buf, header->data, header->size) == 0)) { 269 ret = header->size; 270 vchiq_release_message(service->handle, header); 271 } else { 272 ret = -EFAULT; 273 } 274 } else { 275 vchiq_log_error(vchiq_arm_log_level, 276 "header %pK: bufsize %x < size %x", 277 header, args->bufsize, header->size); 278 WARN(1, "invalid size\n"); 279 ret = -EMSGSIZE; 280 } 281 DEBUG_TRACE(DEQUEUE_MESSAGE_LINE); 282 out: 283 vchiq_service_put(service); 284 return ret; 285 } 286 287 static int vchiq_irq_queue_bulk_tx_rx(struct vchiq_instance *instance, 288 struct vchiq_queue_bulk_transfer *args, 289 enum vchiq_bulk_dir dir, 290 enum vchiq_bulk_mode __user *mode) 291 { 292 struct vchiq_service *service; 293 struct bulk_waiter_node *waiter = NULL; 294 bool found = false; 295 void *userdata; 296 int status = 0; 297 int ret; 298 299 service = find_service_for_instance(instance, args->handle); 300 if (!service) 301 return -EINVAL; 302 303 if (args->mode == VCHIQ_BULK_MODE_BLOCKING) { 304 waiter = kzalloc(sizeof(*waiter), GFP_KERNEL); 305 if (!waiter) { 306 ret = -ENOMEM; 307 goto out; 308 } 309 310 userdata = &waiter->bulk_waiter; 311 } else if (args->mode == VCHIQ_BULK_MODE_WAITING) { 312 mutex_lock(&instance->bulk_waiter_list_mutex); 313 list_for_each_entry(waiter, &instance->bulk_waiter_list, 314 list) { 315 if (waiter->pid == current->pid) { 316 list_del(&waiter->list); 317 found = true; 318 break; 319 } 320 } 321 mutex_unlock(&instance->bulk_waiter_list_mutex); 322 if (!found) { 323 vchiq_log_error(vchiq_arm_log_level, 324 "no bulk_waiter found for pid %d", current->pid); 325 ret = -ESRCH; 326 goto out; 327 } 328 vchiq_log_info(vchiq_arm_log_level, 329 "found bulk_waiter %pK for pid %d", waiter, current->pid); 330 userdata = &waiter->bulk_waiter; 331 } else { 332 userdata = args->userdata; 333 } 334 335 status = vchiq_bulk_transfer(args->handle, NULL, args->data, args->size, 336 userdata, args->mode, dir); 337 338 if (!waiter) { 339 ret = 0; 340 goto out; 341 } 342 343 if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) || 344 !waiter->bulk_waiter.bulk) { 345 if (waiter->bulk_waiter.bulk) { 346 /* Cancel the signal when the transfer completes. */ 347 spin_lock(&bulk_waiter_spinlock); 348 waiter->bulk_waiter.bulk->userdata = NULL; 349 spin_unlock(&bulk_waiter_spinlock); 350 } 351 kfree(waiter); 352 ret = 0; 353 } else { 354 const enum vchiq_bulk_mode mode_waiting = 355 VCHIQ_BULK_MODE_WAITING; 356 waiter->pid = current->pid; 357 mutex_lock(&instance->bulk_waiter_list_mutex); 358 list_add(&waiter->list, &instance->bulk_waiter_list); 359 mutex_unlock(&instance->bulk_waiter_list_mutex); 360 vchiq_log_info(vchiq_arm_log_level, 361 "saved bulk_waiter %pK for pid %d", waiter, current->pid); 362 363 ret = put_user(mode_waiting, mode); 364 } 365 out: 366 vchiq_service_put(service); 367 if (ret) 368 return ret; 369 else if (status == VCHIQ_ERROR) 370 return -EIO; 371 else if (status == VCHIQ_RETRY) 372 return -EINTR; 373 return 0; 374 } 375 376 /* read a user pointer value from an array pointers in user space */ 377 static inline int vchiq_get_user_ptr(void __user **buf, void __user *ubuf, int index) 378 { 379 int ret; 380 381 if (in_compat_syscall()) { 382 compat_uptr_t ptr32; 383 compat_uptr_t __user *uptr = ubuf; 384 385 ret = get_user(ptr32, uptr + index); 386 if (ret) 387 return ret; 388 389 *buf = compat_ptr(ptr32); 390 } else { 391 uintptr_t ptr, __user *uptr = ubuf; 392 393 ret = get_user(ptr, uptr + index); 394 395 if (ret) 396 return ret; 397 398 *buf = (void __user *)ptr; 399 } 400 401 return 0; 402 } 403 404 struct vchiq_completion_data32 { 405 enum vchiq_reason reason; 406 compat_uptr_t header; 407 compat_uptr_t service_userdata; 408 compat_uptr_t bulk_userdata; 409 }; 410 411 static int vchiq_put_completion(struct vchiq_completion_data __user *buf, 412 struct vchiq_completion_data *completion, 413 int index) 414 { 415 struct vchiq_completion_data32 __user *buf32 = (void __user *)buf; 416 417 if (in_compat_syscall()) { 418 struct vchiq_completion_data32 tmp = { 419 .reason = completion->reason, 420 .header = ptr_to_compat(completion->header), 421 .service_userdata = ptr_to_compat(completion->service_userdata), 422 .bulk_userdata = ptr_to_compat(completion->bulk_userdata), 423 }; 424 if (copy_to_user(&buf32[index], &tmp, sizeof(tmp))) 425 return -EFAULT; 426 } else { 427 if (copy_to_user(&buf[index], completion, sizeof(*completion))) 428 return -EFAULT; 429 } 430 431 return 0; 432 } 433 434 static int vchiq_ioc_await_completion(struct vchiq_instance *instance, 435 struct vchiq_await_completion *args, 436 int __user *msgbufcountp) 437 { 438 int msgbufcount; 439 int remove; 440 int ret; 441 442 DEBUG_INITIALISE(g_state.local); 443 444 DEBUG_TRACE(AWAIT_COMPLETION_LINE); 445 if (!instance->connected) 446 return -ENOTCONN; 447 448 mutex_lock(&instance->completion_mutex); 449 450 DEBUG_TRACE(AWAIT_COMPLETION_LINE); 451 while ((instance->completion_remove == instance->completion_insert) && !instance->closing) { 452 int rc; 453 454 DEBUG_TRACE(AWAIT_COMPLETION_LINE); 455 mutex_unlock(&instance->completion_mutex); 456 rc = wait_for_completion_interruptible(&instance->insert_event); 457 mutex_lock(&instance->completion_mutex); 458 if (rc) { 459 DEBUG_TRACE(AWAIT_COMPLETION_LINE); 460 vchiq_log_info(vchiq_arm_log_level, 461 "AWAIT_COMPLETION interrupted"); 462 ret = -EINTR; 463 goto out; 464 } 465 } 466 DEBUG_TRACE(AWAIT_COMPLETION_LINE); 467 468 msgbufcount = args->msgbufcount; 469 remove = instance->completion_remove; 470 471 for (ret = 0; ret < args->count; ret++) { 472 struct vchiq_completion_data_kernel *completion; 473 struct vchiq_completion_data user_completion; 474 struct vchiq_service *service; 475 struct user_service *user_service; 476 struct vchiq_header *header; 477 478 if (remove == instance->completion_insert) 479 break; 480 481 completion = &instance->completions[remove & (MAX_COMPLETIONS - 1)]; 482 483 /* 484 * A read memory barrier is needed to stop 485 * prefetch of a stale completion record 486 */ 487 rmb(); 488 489 service = completion->service_userdata; 490 user_service = service->base.userdata; 491 492 memset(&user_completion, 0, sizeof(user_completion)); 493 user_completion = (struct vchiq_completion_data) { 494 .reason = completion->reason, 495 .service_userdata = user_service->userdata, 496 }; 497 498 header = completion->header; 499 if (header) { 500 void __user *msgbuf; 501 int msglen; 502 503 msglen = header->size + sizeof(struct vchiq_header); 504 /* This must be a VCHIQ-style service */ 505 if (args->msgbufsize < msglen) { 506 vchiq_log_error(vchiq_arm_log_level, 507 "header %pK: msgbufsize %x < msglen %x", 508 header, args->msgbufsize, msglen); 509 WARN(1, "invalid message size\n"); 510 if (ret == 0) 511 ret = -EMSGSIZE; 512 break; 513 } 514 if (msgbufcount <= 0) 515 /* Stall here for lack of a buffer for the message. */ 516 break; 517 /* Get the pointer from user space */ 518 msgbufcount--; 519 if (vchiq_get_user_ptr(&msgbuf, args->msgbufs, 520 msgbufcount)) { 521 if (ret == 0) 522 ret = -EFAULT; 523 break; 524 } 525 526 /* Copy the message to user space */ 527 if (copy_to_user(msgbuf, header, msglen)) { 528 if (ret == 0) 529 ret = -EFAULT; 530 break; 531 } 532 533 /* Now it has been copied, the message can be released. */ 534 vchiq_release_message(service->handle, header); 535 536 /* The completion must point to the msgbuf. */ 537 user_completion.header = msgbuf; 538 } 539 540 if ((completion->reason == VCHIQ_SERVICE_CLOSED) && 541 !instance->use_close_delivered) 542 vchiq_service_put(service); 543 544 /* 545 * FIXME: address space mismatch, does bulk_userdata 546 * actually point to user or kernel memory? 547 */ 548 user_completion.bulk_userdata = completion->bulk_userdata; 549 550 if (vchiq_put_completion(args->buf, &user_completion, ret)) { 551 if (ret == 0) 552 ret = -EFAULT; 553 break; 554 } 555 556 /* 557 * Ensure that the above copy has completed 558 * before advancing the remove pointer. 559 */ 560 mb(); 561 remove++; 562 instance->completion_remove = remove; 563 } 564 565 if (msgbufcount != args->msgbufcount) { 566 if (put_user(msgbufcount, msgbufcountp)) 567 ret = -EFAULT; 568 } 569 out: 570 if (ret) 571 complete(&instance->remove_event); 572 mutex_unlock(&instance->completion_mutex); 573 DEBUG_TRACE(AWAIT_COMPLETION_LINE); 574 575 return ret; 576 } 577 578 static long 579 vchiq_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 580 { 581 struct vchiq_instance *instance = file->private_data; 582 enum vchiq_status status = VCHIQ_SUCCESS; 583 struct vchiq_service *service = NULL; 584 long ret = 0; 585 int i, rc; 586 587 vchiq_log_trace(vchiq_arm_log_level, 588 "%s - instance %pK, cmd %s, arg %lx", __func__, instance, 589 ((_IOC_TYPE(cmd) == VCHIQ_IOC_MAGIC) && (_IOC_NR(cmd) <= VCHIQ_IOC_MAX)) ? 590 ioctl_names[_IOC_NR(cmd)] : "<invalid>", arg); 591 592 switch (cmd) { 593 case VCHIQ_IOC_SHUTDOWN: 594 if (!instance->connected) 595 break; 596 597 /* Remove all services */ 598 i = 0; 599 while ((service = next_service_by_instance(instance->state, 600 instance, &i))) { 601 status = vchiq_remove_service(service->handle); 602 vchiq_service_put(service); 603 if (status != VCHIQ_SUCCESS) 604 break; 605 } 606 service = NULL; 607 608 if (status == VCHIQ_SUCCESS) { 609 /* Wake the completion thread and ask it to exit */ 610 instance->closing = 1; 611 complete(&instance->insert_event); 612 } 613 614 break; 615 616 case VCHIQ_IOC_CONNECT: 617 if (instance->connected) { 618 ret = -EINVAL; 619 break; 620 } 621 rc = mutex_lock_killable(&instance->state->mutex); 622 if (rc) { 623 vchiq_log_error(vchiq_arm_log_level, 624 "vchiq: connect: could not lock mutex for state %d: %d", 625 instance->state->id, rc); 626 ret = -EINTR; 627 break; 628 } 629 status = vchiq_connect_internal(instance->state, instance); 630 mutex_unlock(&instance->state->mutex); 631 632 if (status == VCHIQ_SUCCESS) 633 instance->connected = 1; 634 else 635 vchiq_log_error(vchiq_arm_log_level, 636 "vchiq: could not connect: %d", status); 637 break; 638 639 case VCHIQ_IOC_CREATE_SERVICE: { 640 struct vchiq_create_service __user *argp; 641 struct vchiq_create_service args; 642 643 argp = (void __user *)arg; 644 if (copy_from_user(&args, argp, sizeof(args))) { 645 ret = -EFAULT; 646 break; 647 } 648 649 ret = vchiq_ioc_create_service(instance, &args); 650 if (ret < 0) 651 break; 652 653 if (put_user(args.handle, &argp->handle)) { 654 vchiq_remove_service(args.handle); 655 ret = -EFAULT; 656 } 657 } break; 658 659 case VCHIQ_IOC_CLOSE_SERVICE: 660 case VCHIQ_IOC_REMOVE_SERVICE: { 661 unsigned int handle = (unsigned int)arg; 662 struct user_service *user_service; 663 664 service = find_service_for_instance(instance, handle); 665 if (!service) { 666 ret = -EINVAL; 667 break; 668 } 669 670 user_service = service->base.userdata; 671 672 /* 673 * close_pending is false on first entry, and when the 674 * wait in vchiq_close_service has been interrupted. 675 */ 676 if (!user_service->close_pending) { 677 status = (cmd == VCHIQ_IOC_CLOSE_SERVICE) ? 678 vchiq_close_service(service->handle) : 679 vchiq_remove_service(service->handle); 680 if (status != VCHIQ_SUCCESS) 681 break; 682 } 683 684 /* 685 * close_pending is true once the underlying service 686 * has been closed until the client library calls the 687 * CLOSE_DELIVERED ioctl, signalling close_event. 688 */ 689 if (user_service->close_pending && 690 wait_for_completion_interruptible(&user_service->close_event)) 691 status = VCHIQ_RETRY; 692 break; 693 } 694 695 case VCHIQ_IOC_USE_SERVICE: 696 case VCHIQ_IOC_RELEASE_SERVICE: { 697 unsigned int handle = (unsigned int)arg; 698 699 service = find_service_for_instance(instance, handle); 700 if (service) { 701 ret = (cmd == VCHIQ_IOC_USE_SERVICE) ? 702 vchiq_use_service_internal(service) : 703 vchiq_release_service_internal(service); 704 if (ret) { 705 vchiq_log_error(vchiq_susp_log_level, 706 "%s: cmd %s returned error %ld for service %c%c%c%c:%03d", 707 __func__, (cmd == VCHIQ_IOC_USE_SERVICE) ? 708 "VCHIQ_IOC_USE_SERVICE" : 709 "VCHIQ_IOC_RELEASE_SERVICE", 710 ret, 711 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc), 712 service->client_id); 713 } 714 } else { 715 ret = -EINVAL; 716 } 717 } break; 718 719 case VCHIQ_IOC_QUEUE_MESSAGE: { 720 struct vchiq_queue_message args; 721 722 if (copy_from_user(&args, (const void __user *)arg, 723 sizeof(args))) { 724 ret = -EFAULT; 725 break; 726 } 727 728 service = find_service_for_instance(instance, args.handle); 729 730 if (service && (args.count <= MAX_ELEMENTS)) { 731 /* Copy elements into kernel space */ 732 struct vchiq_element elements[MAX_ELEMENTS]; 733 734 if (copy_from_user(elements, args.elements, 735 args.count * sizeof(struct vchiq_element)) == 0) 736 ret = vchiq_ioc_queue_message(args.handle, elements, 737 args.count); 738 else 739 ret = -EFAULT; 740 } else { 741 ret = -EINVAL; 742 } 743 } break; 744 745 case VCHIQ_IOC_QUEUE_BULK_TRANSMIT: 746 case VCHIQ_IOC_QUEUE_BULK_RECEIVE: { 747 struct vchiq_queue_bulk_transfer args; 748 struct vchiq_queue_bulk_transfer __user *argp; 749 750 enum vchiq_bulk_dir dir = 751 (cmd == VCHIQ_IOC_QUEUE_BULK_TRANSMIT) ? 752 VCHIQ_BULK_TRANSMIT : VCHIQ_BULK_RECEIVE; 753 754 argp = (void __user *)arg; 755 if (copy_from_user(&args, argp, sizeof(args))) { 756 ret = -EFAULT; 757 break; 758 } 759 760 ret = vchiq_irq_queue_bulk_tx_rx(instance, &args, 761 dir, &argp->mode); 762 } break; 763 764 case VCHIQ_IOC_AWAIT_COMPLETION: { 765 struct vchiq_await_completion args; 766 struct vchiq_await_completion __user *argp; 767 768 argp = (void __user *)arg; 769 if (copy_from_user(&args, argp, sizeof(args))) { 770 ret = -EFAULT; 771 break; 772 } 773 774 ret = vchiq_ioc_await_completion(instance, &args, 775 &argp->msgbufcount); 776 } break; 777 778 case VCHIQ_IOC_DEQUEUE_MESSAGE: { 779 struct vchiq_dequeue_message args; 780 781 if (copy_from_user(&args, (const void __user *)arg, 782 sizeof(args))) { 783 ret = -EFAULT; 784 break; 785 } 786 787 ret = vchiq_ioc_dequeue_message(instance, &args); 788 } break; 789 790 case VCHIQ_IOC_GET_CLIENT_ID: { 791 unsigned int handle = (unsigned int)arg; 792 793 ret = vchiq_get_client_id(handle); 794 } break; 795 796 case VCHIQ_IOC_GET_CONFIG: { 797 struct vchiq_get_config args; 798 struct vchiq_config config; 799 800 if (copy_from_user(&args, (const void __user *)arg, 801 sizeof(args))) { 802 ret = -EFAULT; 803 break; 804 } 805 if (args.config_size > sizeof(config)) { 806 ret = -EINVAL; 807 break; 808 } 809 810 vchiq_get_config(&config); 811 if (copy_to_user(args.pconfig, &config, args.config_size)) { 812 ret = -EFAULT; 813 break; 814 } 815 } break; 816 817 case VCHIQ_IOC_SET_SERVICE_OPTION: { 818 struct vchiq_set_service_option args; 819 820 if (copy_from_user(&args, (const void __user *)arg, 821 sizeof(args))) { 822 ret = -EFAULT; 823 break; 824 } 825 826 service = find_service_for_instance(instance, args.handle); 827 if (!service) { 828 ret = -EINVAL; 829 break; 830 } 831 832 ret = vchiq_set_service_option(args.handle, args.option, 833 args.value); 834 } break; 835 836 case VCHIQ_IOC_LIB_VERSION: { 837 unsigned int lib_version = (unsigned int)arg; 838 839 if (lib_version < VCHIQ_VERSION_MIN) 840 ret = -EINVAL; 841 else if (lib_version >= VCHIQ_VERSION_CLOSE_DELIVERED) 842 instance->use_close_delivered = 1; 843 } break; 844 845 case VCHIQ_IOC_CLOSE_DELIVERED: { 846 unsigned int handle = (unsigned int)arg; 847 848 service = find_closed_service_for_instance(instance, handle); 849 if (service) { 850 struct user_service *user_service = 851 (struct user_service *)service->base.userdata; 852 close_delivered(user_service); 853 } else { 854 ret = -EINVAL; 855 } 856 } break; 857 858 default: 859 ret = -ENOTTY; 860 break; 861 } 862 863 if (service) 864 vchiq_service_put(service); 865 866 if (ret == 0) { 867 if (status == VCHIQ_ERROR) 868 ret = -EIO; 869 else if (status == VCHIQ_RETRY) 870 ret = -EINTR; 871 } 872 873 if ((status == VCHIQ_SUCCESS) && (ret < 0) && (ret != -EINTR) && (ret != -EWOULDBLOCK)) 874 vchiq_log_info(vchiq_arm_log_level, 875 " ioctl instance %pK, cmd %s -> status %d, %ld", 876 instance, (_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ? 877 ioctl_names[_IOC_NR(cmd)] : "<invalid>", status, ret); 878 else 879 vchiq_log_trace(vchiq_arm_log_level, 880 " ioctl instance %pK, cmd %s -> status %d, %ld", 881 instance, (_IOC_NR(cmd) <= VCHIQ_IOC_MAX) ? 882 ioctl_names[_IOC_NR(cmd)] : "<invalid>", status, ret); 883 884 return ret; 885 } 886 887 #if defined(CONFIG_COMPAT) 888 889 struct vchiq_service_params32 { 890 int fourcc; 891 compat_uptr_t callback; 892 compat_uptr_t userdata; 893 short version; /* Increment for non-trivial changes */ 894 short version_min; /* Update for incompatible changes */ 895 }; 896 897 struct vchiq_create_service32 { 898 struct vchiq_service_params32 params; 899 int is_open; 900 int is_vchi; 901 unsigned int handle; /* OUT */ 902 }; 903 904 #define VCHIQ_IOC_CREATE_SERVICE32 \ 905 _IOWR(VCHIQ_IOC_MAGIC, 2, struct vchiq_create_service32) 906 907 static long 908 vchiq_compat_ioctl_create_service(struct file *file, unsigned int cmd, 909 struct vchiq_create_service32 __user *ptrargs32) 910 { 911 struct vchiq_create_service args; 912 struct vchiq_create_service32 args32; 913 long ret; 914 915 if (copy_from_user(&args32, ptrargs32, sizeof(args32))) 916 return -EFAULT; 917 918 args = (struct vchiq_create_service) { 919 .params = { 920 .fourcc = args32.params.fourcc, 921 .callback = compat_ptr(args32.params.callback), 922 .userdata = compat_ptr(args32.params.userdata), 923 .version = args32.params.version, 924 .version_min = args32.params.version_min, 925 }, 926 .is_open = args32.is_open, 927 .is_vchi = args32.is_vchi, 928 .handle = args32.handle, 929 }; 930 931 ret = vchiq_ioc_create_service(file->private_data, &args); 932 if (ret < 0) 933 return ret; 934 935 if (put_user(args.handle, &ptrargs32->handle)) { 936 vchiq_remove_service(args.handle); 937 return -EFAULT; 938 } 939 940 return 0; 941 } 942 943 struct vchiq_element32 { 944 compat_uptr_t data; 945 unsigned int size; 946 }; 947 948 struct vchiq_queue_message32 { 949 unsigned int handle; 950 unsigned int count; 951 compat_uptr_t elements; 952 }; 953 954 #define VCHIQ_IOC_QUEUE_MESSAGE32 \ 955 _IOW(VCHIQ_IOC_MAGIC, 4, struct vchiq_queue_message32) 956 957 static long 958 vchiq_compat_ioctl_queue_message(struct file *file, 959 unsigned int cmd, 960 struct vchiq_queue_message32 __user *arg) 961 { 962 struct vchiq_queue_message args; 963 struct vchiq_queue_message32 args32; 964 struct vchiq_service *service; 965 int ret; 966 967 if (copy_from_user(&args32, arg, sizeof(args32))) 968 return -EFAULT; 969 970 args = (struct vchiq_queue_message) { 971 .handle = args32.handle, 972 .count = args32.count, 973 .elements = compat_ptr(args32.elements), 974 }; 975 976 if (args32.count > MAX_ELEMENTS) 977 return -EINVAL; 978 979 service = find_service_for_instance(file->private_data, args.handle); 980 if (!service) 981 return -EINVAL; 982 983 if (args32.elements && args32.count) { 984 struct vchiq_element32 element32[MAX_ELEMENTS]; 985 struct vchiq_element elements[MAX_ELEMENTS]; 986 unsigned int count; 987 988 if (copy_from_user(&element32, args.elements, 989 sizeof(element32))) { 990 vchiq_service_put(service); 991 return -EFAULT; 992 } 993 994 for (count = 0; count < args32.count; count++) { 995 elements[count].data = 996 compat_ptr(element32[count].data); 997 elements[count].size = element32[count].size; 998 } 999 ret = vchiq_ioc_queue_message(args.handle, elements, 1000 args.count); 1001 } else { 1002 ret = -EINVAL; 1003 } 1004 vchiq_service_put(service); 1005 1006 return ret; 1007 } 1008 1009 struct vchiq_queue_bulk_transfer32 { 1010 unsigned int handle; 1011 compat_uptr_t data; 1012 unsigned int size; 1013 compat_uptr_t userdata; 1014 enum vchiq_bulk_mode mode; 1015 }; 1016 1017 #define VCHIQ_IOC_QUEUE_BULK_TRANSMIT32 \ 1018 _IOWR(VCHIQ_IOC_MAGIC, 5, struct vchiq_queue_bulk_transfer32) 1019 #define VCHIQ_IOC_QUEUE_BULK_RECEIVE32 \ 1020 _IOWR(VCHIQ_IOC_MAGIC, 6, struct vchiq_queue_bulk_transfer32) 1021 1022 static long 1023 vchiq_compat_ioctl_queue_bulk(struct file *file, 1024 unsigned int cmd, 1025 struct vchiq_queue_bulk_transfer32 __user *argp) 1026 { 1027 struct vchiq_queue_bulk_transfer32 args32; 1028 struct vchiq_queue_bulk_transfer args; 1029 enum vchiq_bulk_dir dir = (cmd == VCHIQ_IOC_QUEUE_BULK_TRANSMIT32) ? 1030 VCHIQ_BULK_TRANSMIT : VCHIQ_BULK_RECEIVE; 1031 1032 if (copy_from_user(&args32, argp, sizeof(args32))) 1033 return -EFAULT; 1034 1035 args = (struct vchiq_queue_bulk_transfer) { 1036 .handle = args32.handle, 1037 .data = compat_ptr(args32.data), 1038 .size = args32.size, 1039 .userdata = compat_ptr(args32.userdata), 1040 .mode = args32.mode, 1041 }; 1042 1043 return vchiq_irq_queue_bulk_tx_rx(file->private_data, &args, 1044 dir, &argp->mode); 1045 } 1046 1047 struct vchiq_await_completion32 { 1048 unsigned int count; 1049 compat_uptr_t buf; 1050 unsigned int msgbufsize; 1051 unsigned int msgbufcount; /* IN/OUT */ 1052 compat_uptr_t msgbufs; 1053 }; 1054 1055 #define VCHIQ_IOC_AWAIT_COMPLETION32 \ 1056 _IOWR(VCHIQ_IOC_MAGIC, 7, struct vchiq_await_completion32) 1057 1058 static long 1059 vchiq_compat_ioctl_await_completion(struct file *file, 1060 unsigned int cmd, 1061 struct vchiq_await_completion32 __user *argp) 1062 { 1063 struct vchiq_await_completion args; 1064 struct vchiq_await_completion32 args32; 1065 1066 if (copy_from_user(&args32, argp, sizeof(args32))) 1067 return -EFAULT; 1068 1069 args = (struct vchiq_await_completion) { 1070 .count = args32.count, 1071 .buf = compat_ptr(args32.buf), 1072 .msgbufsize = args32.msgbufsize, 1073 .msgbufcount = args32.msgbufcount, 1074 .msgbufs = compat_ptr(args32.msgbufs), 1075 }; 1076 1077 return vchiq_ioc_await_completion(file->private_data, &args, 1078 &argp->msgbufcount); 1079 } 1080 1081 struct vchiq_dequeue_message32 { 1082 unsigned int handle; 1083 int blocking; 1084 unsigned int bufsize; 1085 compat_uptr_t buf; 1086 }; 1087 1088 #define VCHIQ_IOC_DEQUEUE_MESSAGE32 \ 1089 _IOWR(VCHIQ_IOC_MAGIC, 8, struct vchiq_dequeue_message32) 1090 1091 static long 1092 vchiq_compat_ioctl_dequeue_message(struct file *file, 1093 unsigned int cmd, 1094 struct vchiq_dequeue_message32 __user *arg) 1095 { 1096 struct vchiq_dequeue_message32 args32; 1097 struct vchiq_dequeue_message args; 1098 1099 if (copy_from_user(&args32, arg, sizeof(args32))) 1100 return -EFAULT; 1101 1102 args = (struct vchiq_dequeue_message) { 1103 .handle = args32.handle, 1104 .blocking = args32.blocking, 1105 .bufsize = args32.bufsize, 1106 .buf = compat_ptr(args32.buf), 1107 }; 1108 1109 return vchiq_ioc_dequeue_message(file->private_data, &args); 1110 } 1111 1112 struct vchiq_get_config32 { 1113 unsigned int config_size; 1114 compat_uptr_t pconfig; 1115 }; 1116 1117 #define VCHIQ_IOC_GET_CONFIG32 \ 1118 _IOWR(VCHIQ_IOC_MAGIC, 10, struct vchiq_get_config32) 1119 1120 static long 1121 vchiq_compat_ioctl_get_config(struct file *file, 1122 unsigned int cmd, 1123 struct vchiq_get_config32 __user *arg) 1124 { 1125 struct vchiq_get_config32 args32; 1126 struct vchiq_config config; 1127 void __user *ptr; 1128 1129 if (copy_from_user(&args32, arg, sizeof(args32))) 1130 return -EFAULT; 1131 if (args32.config_size > sizeof(config)) 1132 return -EINVAL; 1133 1134 vchiq_get_config(&config); 1135 ptr = compat_ptr(args32.pconfig); 1136 if (copy_to_user(ptr, &config, args32.config_size)) 1137 return -EFAULT; 1138 1139 return 0; 1140 } 1141 1142 static long 1143 vchiq_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 1144 { 1145 void __user *argp = compat_ptr(arg); 1146 1147 switch (cmd) { 1148 case VCHIQ_IOC_CREATE_SERVICE32: 1149 return vchiq_compat_ioctl_create_service(file, cmd, argp); 1150 case VCHIQ_IOC_QUEUE_MESSAGE32: 1151 return vchiq_compat_ioctl_queue_message(file, cmd, argp); 1152 case VCHIQ_IOC_QUEUE_BULK_TRANSMIT32: 1153 case VCHIQ_IOC_QUEUE_BULK_RECEIVE32: 1154 return vchiq_compat_ioctl_queue_bulk(file, cmd, argp); 1155 case VCHIQ_IOC_AWAIT_COMPLETION32: 1156 return vchiq_compat_ioctl_await_completion(file, cmd, argp); 1157 case VCHIQ_IOC_DEQUEUE_MESSAGE32: 1158 return vchiq_compat_ioctl_dequeue_message(file, cmd, argp); 1159 case VCHIQ_IOC_GET_CONFIG32: 1160 return vchiq_compat_ioctl_get_config(file, cmd, argp); 1161 default: 1162 return vchiq_ioctl(file, cmd, (unsigned long)argp); 1163 } 1164 } 1165 1166 #endif 1167 1168 static int vchiq_open(struct inode *inode, struct file *file) 1169 { 1170 struct vchiq_state *state = vchiq_get_state(); 1171 struct vchiq_instance *instance; 1172 1173 vchiq_log_info(vchiq_arm_log_level, "vchiq_open"); 1174 1175 if (!state) { 1176 vchiq_log_error(vchiq_arm_log_level, 1177 "vchiq has no connection to VideoCore"); 1178 return -ENOTCONN; 1179 } 1180 1181 instance = kzalloc(sizeof(*instance), GFP_KERNEL); 1182 if (!instance) 1183 return -ENOMEM; 1184 1185 instance->state = state; 1186 instance->pid = current->tgid; 1187 1188 vchiq_debugfs_add_instance(instance); 1189 1190 init_completion(&instance->insert_event); 1191 init_completion(&instance->remove_event); 1192 mutex_init(&instance->completion_mutex); 1193 mutex_init(&instance->bulk_waiter_list_mutex); 1194 INIT_LIST_HEAD(&instance->bulk_waiter_list); 1195 1196 file->private_data = instance; 1197 1198 return 0; 1199 } 1200 1201 static int vchiq_release(struct inode *inode, struct file *file) 1202 { 1203 struct vchiq_instance *instance = file->private_data; 1204 struct vchiq_state *state = vchiq_get_state(); 1205 struct vchiq_service *service; 1206 int ret = 0; 1207 int i; 1208 1209 vchiq_log_info(vchiq_arm_log_level, "%s: instance=%lx", __func__, 1210 (unsigned long)instance); 1211 1212 if (!state) { 1213 ret = -EPERM; 1214 goto out; 1215 } 1216 1217 /* Ensure videocore is awake to allow termination. */ 1218 vchiq_use_internal(instance->state, NULL, USE_TYPE_VCHIQ); 1219 1220 mutex_lock(&instance->completion_mutex); 1221 1222 /* Wake the completion thread and ask it to exit */ 1223 instance->closing = 1; 1224 complete(&instance->insert_event); 1225 1226 mutex_unlock(&instance->completion_mutex); 1227 1228 /* Wake the slot handler if the completion queue is full. */ 1229 complete(&instance->remove_event); 1230 1231 /* Mark all services for termination... */ 1232 i = 0; 1233 while ((service = next_service_by_instance(state, instance, &i))) { 1234 struct user_service *user_service = service->base.userdata; 1235 1236 /* Wake the slot handler if the msg queue is full. */ 1237 complete(&user_service->remove_event); 1238 1239 vchiq_terminate_service_internal(service); 1240 vchiq_service_put(service); 1241 } 1242 1243 /* ...and wait for them to die */ 1244 i = 0; 1245 while ((service = next_service_by_instance(state, instance, &i))) { 1246 struct user_service *user_service = service->base.userdata; 1247 1248 wait_for_completion(&service->remove_event); 1249 1250 if (WARN_ON(service->srvstate != VCHIQ_SRVSTATE_FREE)) { 1251 vchiq_service_put(service); 1252 break; 1253 } 1254 1255 spin_lock(&msg_queue_spinlock); 1256 1257 while (user_service->msg_remove != user_service->msg_insert) { 1258 struct vchiq_header *header; 1259 int m = user_service->msg_remove & (MSG_QUEUE_SIZE - 1); 1260 1261 header = user_service->msg_queue[m]; 1262 user_service->msg_remove++; 1263 spin_unlock(&msg_queue_spinlock); 1264 1265 if (header) 1266 vchiq_release_message(service->handle, header); 1267 spin_lock(&msg_queue_spinlock); 1268 } 1269 1270 spin_unlock(&msg_queue_spinlock); 1271 1272 vchiq_service_put(service); 1273 } 1274 1275 /* Release any closed services */ 1276 while (instance->completion_remove != instance->completion_insert) { 1277 struct vchiq_completion_data_kernel *completion; 1278 struct vchiq_service *service; 1279 1280 completion = &instance->completions[instance->completion_remove 1281 & (MAX_COMPLETIONS - 1)]; 1282 service = completion->service_userdata; 1283 if (completion->reason == VCHIQ_SERVICE_CLOSED) { 1284 struct user_service *user_service = 1285 service->base.userdata; 1286 1287 /* Wake any blocked user-thread */ 1288 if (instance->use_close_delivered) 1289 complete(&user_service->close_event); 1290 vchiq_service_put(service); 1291 } 1292 instance->completion_remove++; 1293 } 1294 1295 /* Release the PEER service count. */ 1296 vchiq_release_internal(instance->state, NULL); 1297 1298 free_bulk_waiter(instance); 1299 1300 vchiq_debugfs_remove_instance(instance); 1301 1302 kfree(instance); 1303 file->private_data = NULL; 1304 1305 out: 1306 return ret; 1307 } 1308 1309 static ssize_t 1310 vchiq_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 1311 { 1312 struct dump_context context; 1313 int err; 1314 1315 context.buf = buf; 1316 context.actual = 0; 1317 context.space = count; 1318 context.offset = *ppos; 1319 1320 err = vchiq_dump_state(&context, &g_state); 1321 if (err) 1322 return err; 1323 1324 *ppos += context.actual; 1325 1326 return context.actual; 1327 } 1328 1329 static const struct file_operations 1330 vchiq_fops = { 1331 .owner = THIS_MODULE, 1332 .unlocked_ioctl = vchiq_ioctl, 1333 #if defined(CONFIG_COMPAT) 1334 .compat_ioctl = vchiq_compat_ioctl, 1335 #endif 1336 .open = vchiq_open, 1337 .release = vchiq_release, 1338 .read = vchiq_read 1339 }; 1340 1341 static struct miscdevice vchiq_miscdev = { 1342 .fops = &vchiq_fops, 1343 .minor = MISC_DYNAMIC_MINOR, 1344 .name = "vchiq", 1345 1346 }; 1347 1348 /** 1349 * vchiq_register_chrdev - Register the char driver for vchiq 1350 * and create the necessary class and 1351 * device files in userspace. 1352 * @parent The parent of the char device. 1353 * 1354 * Returns 0 on success else returns the error code. 1355 */ 1356 int vchiq_register_chrdev(struct device *parent) 1357 { 1358 vchiq_miscdev.parent = parent; 1359 1360 return misc_register(&vchiq_miscdev); 1361 } 1362 1363 /** 1364 * vchiq_deregister_chrdev - Deregister and cleanup the vchiq char 1365 * driver and device files 1366 */ 1367 void vchiq_deregister_chrdev(void) 1368 { 1369 misc_deregister(&vchiq_miscdev); 1370 } 1371