1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */ 3 4 #ifndef VCHIQ_CORE_H 5 #define VCHIQ_CORE_H 6 7 #include <linux/mutex.h> 8 #include <linux/completion.h> 9 #include <linux/kthread.h> 10 #include <linux/kref.h> 11 #include <linux/rcupdate.h> 12 #include <linux/wait.h> 13 #include <linux/raspberrypi/vchiq.h> 14 15 #include "vchiq_cfg.h" 16 17 18 /* Do this so that we can test-build the code on non-rpi systems */ 19 #if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) 20 21 #else 22 23 #ifndef dsb 24 #define dsb(a) 25 #endif 26 27 #endif /* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */ 28 29 #define VCHIQ_SERVICE_HANDLE_INVALID 0 30 31 #define VCHIQ_SLOT_SIZE 4096 32 #define VCHIQ_MAX_MSG_SIZE (VCHIQ_SLOT_SIZE - sizeof(struct vchiq_header)) 33 34 /* Run time control of log level, based on KERN_XXX level. */ 35 #define VCHIQ_LOG_DEFAULT 4 36 #define VCHIQ_LOG_ERROR 3 37 #define VCHIQ_LOG_WARNING 4 38 #define VCHIQ_LOG_INFO 6 39 #define VCHIQ_LOG_TRACE 7 40 41 #define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: " 42 43 #ifndef vchiq_log_error 44 #define vchiq_log_error(cat, fmt, ...) \ 45 do { if (cat >= VCHIQ_LOG_ERROR) \ 46 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 47 #endif 48 #ifndef vchiq_log_warning 49 #define vchiq_log_warning(cat, fmt, ...) \ 50 do { if (cat >= VCHIQ_LOG_WARNING) \ 51 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 52 #endif 53 #ifndef vchiq_log_info 54 #define vchiq_log_info(cat, fmt, ...) \ 55 do { if (cat >= VCHIQ_LOG_INFO) \ 56 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 57 #endif 58 #ifndef vchiq_log_trace 59 #define vchiq_log_trace(cat, fmt, ...) \ 60 do { if (cat >= VCHIQ_LOG_TRACE) \ 61 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 62 #endif 63 64 #define vchiq_loud_error(...) \ 65 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__) 66 67 #define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1) 68 #define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1) 69 #define VCHIQ_SLOT_ZERO_SLOTS DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \ 70 VCHIQ_SLOT_SIZE) 71 72 #define VCHIQ_FOURCC_AS_4CHARS(fourcc) \ 73 ((fourcc) >> 24) & 0xff, \ 74 ((fourcc) >> 16) & 0xff, \ 75 ((fourcc) >> 8) & 0xff, \ 76 (fourcc) & 0xff 77 78 typedef uint32_t BITSET_T; 79 80 static_assert((sizeof(BITSET_T) * 8) == 32); 81 82 #define BITSET_SIZE(b) ((b + 31) >> 5) 83 #define BITSET_WORD(b) (b >> 5) 84 #define BITSET_BIT(b) (1 << (b & 31)) 85 #define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b)) 86 #define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b)) 87 #define BITSET_CLR(bs, b) (bs[BITSET_WORD(b)] &= ~BITSET_BIT(b)) 88 89 enum { 90 DEBUG_ENTRIES, 91 #if VCHIQ_ENABLE_DEBUG 92 DEBUG_SLOT_HANDLER_COUNT, 93 DEBUG_SLOT_HANDLER_LINE, 94 DEBUG_PARSE_LINE, 95 DEBUG_PARSE_HEADER, 96 DEBUG_PARSE_MSGID, 97 DEBUG_AWAIT_COMPLETION_LINE, 98 DEBUG_DEQUEUE_MESSAGE_LINE, 99 DEBUG_SERVICE_CALLBACK_LINE, 100 DEBUG_MSG_QUEUE_FULL_COUNT, 101 DEBUG_COMPLETION_QUEUE_FULL_COUNT, 102 #endif 103 DEBUG_MAX 104 }; 105 106 #if VCHIQ_ENABLE_DEBUG 107 108 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug; 109 #define DEBUG_TRACE(d) \ 110 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0) 111 #define DEBUG_VALUE(d, v) \ 112 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0) 113 #define DEBUG_COUNT(d) \ 114 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0) 115 116 #else /* VCHIQ_ENABLE_DEBUG */ 117 118 #define DEBUG_INITIALISE(local) 119 #define DEBUG_TRACE(d) 120 #define DEBUG_VALUE(d, v) 121 #define DEBUG_COUNT(d) 122 123 #endif /* VCHIQ_ENABLE_DEBUG */ 124 125 enum vchiq_connstate { 126 VCHIQ_CONNSTATE_DISCONNECTED, 127 VCHIQ_CONNSTATE_CONNECTING, 128 VCHIQ_CONNSTATE_CONNECTED, 129 VCHIQ_CONNSTATE_PAUSING, 130 VCHIQ_CONNSTATE_PAUSE_SENT, 131 VCHIQ_CONNSTATE_PAUSED, 132 VCHIQ_CONNSTATE_RESUMING, 133 VCHIQ_CONNSTATE_PAUSE_TIMEOUT, 134 VCHIQ_CONNSTATE_RESUME_TIMEOUT 135 }; 136 137 enum { 138 VCHIQ_SRVSTATE_FREE, 139 VCHIQ_SRVSTATE_HIDDEN, 140 VCHIQ_SRVSTATE_LISTENING, 141 VCHIQ_SRVSTATE_OPENING, 142 VCHIQ_SRVSTATE_OPEN, 143 VCHIQ_SRVSTATE_OPENSYNC, 144 VCHIQ_SRVSTATE_CLOSESENT, 145 VCHIQ_SRVSTATE_CLOSERECVD, 146 VCHIQ_SRVSTATE_CLOSEWAIT, 147 VCHIQ_SRVSTATE_CLOSED 148 }; 149 150 enum vchiq_bulk_dir { 151 VCHIQ_BULK_TRANSMIT, 152 VCHIQ_BULK_RECEIVE 153 }; 154 155 typedef void (*vchiq_userdata_term)(void *userdata); 156 157 struct vchiq_bulk { 158 short mode; 159 short dir; 160 void *userdata; 161 dma_addr_t data; 162 int size; 163 void *remote_data; 164 int remote_size; 165 int actual; 166 }; 167 168 struct vchiq_bulk_queue { 169 int local_insert; /* Where to insert the next local bulk */ 170 int remote_insert; /* Where to insert the next remote bulk (master) */ 171 int process; /* Bulk to transfer next */ 172 int remote_notify; /* Bulk to notify the remote client of next (mstr) */ 173 int remove; /* Bulk to notify the local client of, and remove, next */ 174 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS]; 175 }; 176 177 struct remote_event { 178 int armed; 179 int fired; 180 u32 __unused; 181 }; 182 183 struct opaque_platform_state; 184 185 struct vchiq_slot { 186 char data[VCHIQ_SLOT_SIZE]; 187 }; 188 189 struct vchiq_slot_info { 190 /* Use two counters rather than one to avoid the need for a mutex. */ 191 short use_count; 192 short release_count; 193 }; 194 195 struct vchiq_service { 196 struct vchiq_service_base base; 197 unsigned int handle; 198 struct kref ref_count; 199 struct rcu_head rcu; 200 int srvstate; 201 vchiq_userdata_term userdata_term; 202 unsigned int localport; 203 unsigned int remoteport; 204 int public_fourcc; 205 int client_id; 206 char auto_close; 207 char sync; 208 char closing; 209 char trace; 210 atomic_t poll_flags; 211 short version; 212 short version_min; 213 short peer_version; 214 215 struct vchiq_state *state; 216 struct vchiq_instance *instance; 217 218 int service_use_count; 219 220 struct vchiq_bulk_queue bulk_tx; 221 struct vchiq_bulk_queue bulk_rx; 222 223 struct completion remove_event; 224 struct completion bulk_remove_event; 225 struct mutex bulk_mutex; 226 227 struct service_stats_struct { 228 int quota_stalls; 229 int slot_stalls; 230 int bulk_stalls; 231 int error_count; 232 int ctrl_tx_count; 233 int ctrl_rx_count; 234 int bulk_tx_count; 235 int bulk_rx_count; 236 int bulk_aborted_count; 237 uint64_t ctrl_tx_bytes; 238 uint64_t ctrl_rx_bytes; 239 uint64_t bulk_tx_bytes; 240 uint64_t bulk_rx_bytes; 241 } stats; 242 243 int msg_queue_read; 244 int msg_queue_write; 245 struct completion msg_queue_pop; 246 struct completion msg_queue_push; 247 struct vchiq_header *msg_queue[VCHIQ_MAX_SLOTS]; 248 }; 249 250 /* 251 * The quota information is outside struct vchiq_service so that it can 252 * be statically allocated, since for accounting reasons a service's slot 253 * usage is carried over between users of the same port number. 254 */ 255 struct vchiq_service_quota { 256 unsigned short slot_quota; 257 unsigned short slot_use_count; 258 unsigned short message_quota; 259 unsigned short message_use_count; 260 struct completion quota_event; 261 int previous_tx_index; 262 }; 263 264 struct vchiq_shared_state { 265 266 /* A non-zero value here indicates that the content is valid. */ 267 int initialised; 268 269 /* The first and last (inclusive) slots allocated to the owner. */ 270 int slot_first; 271 int slot_last; 272 273 /* The slot allocated to synchronous messages from the owner. */ 274 int slot_sync; 275 276 /* 277 * Signalling this event indicates that owner's slot handler thread 278 * should run. 279 */ 280 struct remote_event trigger; 281 282 /* 283 * Indicates the byte position within the stream where the next message 284 * will be written. The least significant bits are an index into the 285 * slot. The next bits are the index of the slot in slot_queue. 286 */ 287 int tx_pos; 288 289 /* This event should be signalled when a slot is recycled. */ 290 struct remote_event recycle; 291 292 /* The slot_queue index where the next recycled slot will be written. */ 293 int slot_queue_recycle; 294 295 /* This event should be signalled when a synchronous message is sent. */ 296 struct remote_event sync_trigger; 297 298 /* 299 * This event should be signalled when a synchronous message has been 300 * released. 301 */ 302 struct remote_event sync_release; 303 304 /* A circular buffer of slot indexes. */ 305 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE]; 306 307 /* Debugging state */ 308 int debug[DEBUG_MAX]; 309 }; 310 311 struct vchiq_slot_zero { 312 int magic; 313 short version; 314 short version_min; 315 int slot_zero_size; 316 int slot_size; 317 int max_slots; 318 int max_slots_per_side; 319 int platform_data[2]; 320 struct vchiq_shared_state master; 321 struct vchiq_shared_state slave; 322 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS]; 323 }; 324 325 struct vchiq_state { 326 int id; 327 int initialised; 328 enum vchiq_connstate conn_state; 329 short version_common; 330 331 struct vchiq_shared_state *local; 332 struct vchiq_shared_state *remote; 333 struct vchiq_slot *slot_data; 334 335 unsigned short default_slot_quota; 336 unsigned short default_message_quota; 337 338 /* Event indicating connect message received */ 339 struct completion connect; 340 341 /* Mutex protecting services */ 342 struct mutex mutex; 343 struct vchiq_instance **instance; 344 345 /* Processes incoming messages */ 346 struct task_struct *slot_handler_thread; 347 348 /* Processes recycled slots */ 349 struct task_struct *recycle_thread; 350 351 /* Processes synchronous messages */ 352 struct task_struct *sync_thread; 353 354 /* Local implementation of the trigger remote event */ 355 wait_queue_head_t trigger_event; 356 357 /* Local implementation of the recycle remote event */ 358 wait_queue_head_t recycle_event; 359 360 /* Local implementation of the sync trigger remote event */ 361 wait_queue_head_t sync_trigger_event; 362 363 /* Local implementation of the sync release remote event */ 364 wait_queue_head_t sync_release_event; 365 366 char *tx_data; 367 char *rx_data; 368 struct vchiq_slot_info *rx_info; 369 370 struct mutex slot_mutex; 371 372 struct mutex recycle_mutex; 373 374 struct mutex sync_mutex; 375 376 struct mutex bulk_transfer_mutex; 377 378 /* 379 * Indicates the byte position within the stream from where the next 380 * message will be read. The least significant bits are an index into 381 * the slot.The next bits are the index of the slot in 382 * remote->slot_queue. 383 */ 384 int rx_pos; 385 386 /* 387 * A cached copy of local->tx_pos. Only write to local->tx_pos, and read 388 * from remote->tx_pos. 389 */ 390 int local_tx_pos; 391 392 /* The slot_queue index of the slot to become available next. */ 393 int slot_queue_available; 394 395 /* A flag to indicate if any poll has been requested */ 396 int poll_needed; 397 398 /* Ths index of the previous slot used for data messages. */ 399 int previous_data_index; 400 401 /* The number of slots occupied by data messages. */ 402 unsigned short data_use_count; 403 404 /* The maximum number of slots to be occupied by data messages. */ 405 unsigned short data_quota; 406 407 /* An array of bit sets indicating which services must be polled. */ 408 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)]; 409 410 /* The number of the first unused service */ 411 int unused_service; 412 413 /* Signalled when a free slot becomes available. */ 414 struct completion slot_available_event; 415 416 struct completion slot_remove_event; 417 418 /* Signalled when a free data slot becomes available. */ 419 struct completion data_quota_event; 420 421 struct state_stats_struct { 422 int slot_stalls; 423 int data_stalls; 424 int ctrl_tx_count; 425 int ctrl_rx_count; 426 int error_count; 427 } stats; 428 429 struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES]; 430 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES]; 431 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS]; 432 433 struct opaque_platform_state *platform_state; 434 }; 435 436 struct bulk_waiter { 437 struct vchiq_bulk *bulk; 438 struct completion event; 439 int actual; 440 }; 441 442 struct vchiq_config { 443 unsigned int max_msg_size; 444 unsigned int bulk_threshold; /* The message size above which it 445 * is better to use a bulk transfer 446 * (<= max_msg_size) 447 */ 448 unsigned int max_outstanding_bulks; 449 unsigned int max_services; 450 short version; /* The version of VCHIQ */ 451 short version_min; /* The minimum compatible version of VCHIQ */ 452 }; 453 454 455 extern spinlock_t bulk_waiter_spinlock; 456 457 extern int vchiq_core_log_level; 458 extern int vchiq_core_msg_log_level; 459 extern int vchiq_sync_log_level; 460 461 extern struct vchiq_state *vchiq_states[VCHIQ_MAX_STATES]; 462 463 extern const char * 464 get_conn_state_name(enum vchiq_connstate conn_state); 465 466 extern struct vchiq_slot_zero * 467 vchiq_init_slots(void *mem_base, int mem_size); 468 469 extern int 470 vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero); 471 472 extern enum vchiq_status 473 vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance); 474 475 struct vchiq_service * 476 vchiq_add_service_internal(struct vchiq_state *state, 477 const struct vchiq_service_params_kernel *params, 478 int srvstate, struct vchiq_instance *instance, 479 vchiq_userdata_term userdata_term); 480 481 extern enum vchiq_status 482 vchiq_open_service_internal(struct vchiq_service *service, int client_id); 483 484 extern enum vchiq_status 485 vchiq_close_service_internal(struct vchiq_service *service, int close_recvd); 486 487 extern void 488 vchiq_terminate_service_internal(struct vchiq_service *service); 489 490 extern void 491 vchiq_free_service_internal(struct vchiq_service *service); 492 493 extern void 494 vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance); 495 496 extern void 497 remote_event_pollall(struct vchiq_state *state); 498 499 extern enum vchiq_status 500 vchiq_bulk_transfer(unsigned int handle, void *offset, void __user *uoffset, 501 int size, void *userdata, enum vchiq_bulk_mode mode, 502 enum vchiq_bulk_dir dir); 503 504 extern int 505 vchiq_dump_state(void *dump_context, struct vchiq_state *state); 506 507 extern int 508 vchiq_dump_service_state(void *dump_context, struct vchiq_service *service); 509 510 extern void 511 vchiq_loud_error_header(void); 512 513 extern void 514 vchiq_loud_error_footer(void); 515 516 extern void 517 request_poll(struct vchiq_state *state, struct vchiq_service *service, 518 int poll_type); 519 520 static inline struct vchiq_service * 521 handle_to_service(unsigned int handle) 522 { 523 int idx = handle & (VCHIQ_MAX_SERVICES - 1); 524 struct vchiq_state *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) & 525 (VCHIQ_MAX_STATES - 1)]; 526 527 if (!state) 528 return NULL; 529 return rcu_dereference(state->services[idx]); 530 } 531 532 extern struct vchiq_service * 533 find_service_by_handle(unsigned int handle); 534 535 extern struct vchiq_service * 536 find_service_by_port(struct vchiq_state *state, int localport); 537 538 extern struct vchiq_service * 539 find_service_for_instance(struct vchiq_instance *instance, 540 unsigned int handle); 541 542 extern struct vchiq_service * 543 find_closed_service_for_instance(struct vchiq_instance *instance, 544 unsigned int handle); 545 546 extern struct vchiq_service * 547 __next_service_by_instance(struct vchiq_state *state, 548 struct vchiq_instance *instance, 549 int *pidx); 550 551 extern struct vchiq_service * 552 next_service_by_instance(struct vchiq_state *state, 553 struct vchiq_instance *instance, 554 int *pidx); 555 556 extern void 557 vchiq_service_get(struct vchiq_service *service); 558 559 extern void 560 vchiq_service_put(struct vchiq_service *service); 561 562 extern enum vchiq_status 563 vchiq_queue_message(unsigned int handle, 564 ssize_t (*copy_callback)(void *context, void *dest, 565 size_t offset, size_t maxsize), 566 void *context, 567 size_t size); 568 569 /* 570 * The following functions are called from vchiq_core, and external 571 * implementations must be provided. 572 */ 573 574 extern int 575 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, 576 void __user *uoffset, int size, int dir); 577 578 extern void 579 vchiq_complete_bulk(struct vchiq_bulk *bulk); 580 581 extern void 582 remote_event_signal(struct remote_event *event); 583 584 extern int 585 vchiq_dump(void *dump_context, const char *str, int len); 586 587 extern int 588 vchiq_dump_platform_state(void *dump_context); 589 590 extern int 591 vchiq_dump_platform_instances(void *dump_context); 592 593 extern int 594 vchiq_dump_platform_service_state(void *dump_context, 595 struct vchiq_service *service); 596 597 extern int 598 vchiq_use_service_internal(struct vchiq_service *service); 599 600 extern int 601 vchiq_release_service_internal(struct vchiq_service *service); 602 603 extern void 604 vchiq_on_remote_use(struct vchiq_state *state); 605 606 extern void 607 vchiq_on_remote_release(struct vchiq_state *state); 608 609 extern int 610 vchiq_platform_init_state(struct vchiq_state *state); 611 612 extern enum vchiq_status 613 vchiq_check_service(struct vchiq_service *service); 614 615 extern void 616 vchiq_on_remote_use_active(struct vchiq_state *state); 617 618 extern enum vchiq_status 619 vchiq_send_remote_use(struct vchiq_state *state); 620 621 extern enum vchiq_status 622 vchiq_send_remote_use_active(struct vchiq_state *state); 623 624 extern void 625 vchiq_platform_conn_state_changed(struct vchiq_state *state, 626 enum vchiq_connstate oldstate, 627 enum vchiq_connstate newstate); 628 629 extern void 630 vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate); 631 632 extern void 633 vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem, 634 size_t numBytes); 635 636 extern enum vchiq_status vchiq_remove_service(unsigned int service); 637 638 extern int vchiq_get_client_id(unsigned int service); 639 640 extern void vchiq_get_config(struct vchiq_config *config); 641 642 extern int 643 vchiq_set_service_option(unsigned int service, enum vchiq_service_option option, 644 int value); 645 646 #endif 647