1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */ 3 4 #ifndef VCHIQ_CORE_H 5 #define VCHIQ_CORE_H 6 7 #include <linux/mutex.h> 8 #include <linux/completion.h> 9 #include <linux/kthread.h> 10 #include <linux/kref.h> 11 #include <linux/rcupdate.h> 12 #include <linux/wait.h> 13 #include <linux/raspberrypi/vchiq.h> 14 15 #include "vchiq_cfg.h" 16 17 /* Do this so that we can test-build the code on non-rpi systems */ 18 #if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) 19 20 #else 21 22 #ifndef dsb 23 #define dsb(a) 24 #endif 25 26 #endif /* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */ 27 28 #define VCHIQ_SERVICE_HANDLE_INVALID 0 29 30 #define VCHIQ_SLOT_SIZE 4096 31 #define VCHIQ_MAX_MSG_SIZE (VCHIQ_SLOT_SIZE - sizeof(struct vchiq_header)) 32 33 /* Run time control of log level, based on KERN_XXX level. */ 34 #define VCHIQ_LOG_DEFAULT 4 35 #define VCHIQ_LOG_ERROR 3 36 #define VCHIQ_LOG_WARNING 4 37 #define VCHIQ_LOG_INFO 6 38 #define VCHIQ_LOG_TRACE 7 39 40 #define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: " 41 42 #ifndef vchiq_log_error 43 #define vchiq_log_error(cat, fmt, ...) \ 44 do { if (cat >= VCHIQ_LOG_ERROR) \ 45 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 46 #endif 47 #ifndef vchiq_log_warning 48 #define vchiq_log_warning(cat, fmt, ...) \ 49 do { if (cat >= VCHIQ_LOG_WARNING) \ 50 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 51 #endif 52 #ifndef vchiq_log_info 53 #define vchiq_log_info(cat, fmt, ...) \ 54 do { if (cat >= VCHIQ_LOG_INFO) \ 55 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 56 #endif 57 #ifndef vchiq_log_trace 58 #define vchiq_log_trace(cat, fmt, ...) \ 59 do { if (cat >= VCHIQ_LOG_TRACE) \ 60 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 61 #endif 62 63 #define vchiq_loud_error(...) \ 64 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__) 65 66 #define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1) 67 #define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1) 68 #define VCHIQ_SLOT_ZERO_SLOTS DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \ 69 VCHIQ_SLOT_SIZE) 70 71 #define VCHIQ_FOURCC_AS_4CHARS(fourcc) \ 72 ((fourcc) >> 24) & 0xff, \ 73 ((fourcc) >> 16) & 0xff, \ 74 ((fourcc) >> 8) & 0xff, \ 75 (fourcc) & 0xff 76 77 static_assert((sizeof(u32) * 8) == 32); 78 79 #define BITSET_SIZE(b) ((b + 31) >> 5) 80 #define BITSET_WORD(b) (b >> 5) 81 #define BITSET_BIT(b) (1 << (b & 31)) 82 #define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b)) 83 #define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b)) 84 #define BITSET_CLR(bs, b) (bs[BITSET_WORD(b)] &= ~BITSET_BIT(b)) 85 86 enum { 87 DEBUG_ENTRIES, 88 #if VCHIQ_ENABLE_DEBUG 89 DEBUG_SLOT_HANDLER_COUNT, 90 DEBUG_SLOT_HANDLER_LINE, 91 DEBUG_PARSE_LINE, 92 DEBUG_PARSE_HEADER, 93 DEBUG_PARSE_MSGID, 94 DEBUG_AWAIT_COMPLETION_LINE, 95 DEBUG_DEQUEUE_MESSAGE_LINE, 96 DEBUG_SERVICE_CALLBACK_LINE, 97 DEBUG_MSG_QUEUE_FULL_COUNT, 98 DEBUG_COMPLETION_QUEUE_FULL_COUNT, 99 #endif 100 DEBUG_MAX 101 }; 102 103 #if VCHIQ_ENABLE_DEBUG 104 105 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug 106 #define DEBUG_TRACE(d) \ 107 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0) 108 #define DEBUG_VALUE(d, v) \ 109 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0) 110 #define DEBUG_COUNT(d) \ 111 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0) 112 113 #else /* VCHIQ_ENABLE_DEBUG */ 114 115 #define DEBUG_INITIALISE(local) 116 #define DEBUG_TRACE(d) 117 #define DEBUG_VALUE(d, v) 118 #define DEBUG_COUNT(d) 119 120 #endif /* VCHIQ_ENABLE_DEBUG */ 121 122 enum vchiq_connstate { 123 VCHIQ_CONNSTATE_DISCONNECTED, 124 VCHIQ_CONNSTATE_CONNECTING, 125 VCHIQ_CONNSTATE_CONNECTED, 126 VCHIQ_CONNSTATE_PAUSING, 127 VCHIQ_CONNSTATE_PAUSE_SENT, 128 VCHIQ_CONNSTATE_PAUSED, 129 VCHIQ_CONNSTATE_RESUMING, 130 VCHIQ_CONNSTATE_PAUSE_TIMEOUT, 131 VCHIQ_CONNSTATE_RESUME_TIMEOUT 132 }; 133 134 enum { 135 VCHIQ_SRVSTATE_FREE, 136 VCHIQ_SRVSTATE_HIDDEN, 137 VCHIQ_SRVSTATE_LISTENING, 138 VCHIQ_SRVSTATE_OPENING, 139 VCHIQ_SRVSTATE_OPEN, 140 VCHIQ_SRVSTATE_OPENSYNC, 141 VCHIQ_SRVSTATE_CLOSESENT, 142 VCHIQ_SRVSTATE_CLOSERECVD, 143 VCHIQ_SRVSTATE_CLOSEWAIT, 144 VCHIQ_SRVSTATE_CLOSED 145 }; 146 147 enum vchiq_bulk_dir { 148 VCHIQ_BULK_TRANSMIT, 149 VCHIQ_BULK_RECEIVE 150 }; 151 152 struct vchiq_bulk { 153 short mode; 154 short dir; 155 void *userdata; 156 dma_addr_t data; 157 int size; 158 void *remote_data; 159 int remote_size; 160 int actual; 161 }; 162 163 struct vchiq_bulk_queue { 164 int local_insert; /* Where to insert the next local bulk */ 165 int remote_insert; /* Where to insert the next remote bulk (master) */ 166 int process; /* Bulk to transfer next */ 167 int remote_notify; /* Bulk to notify the remote client of next (mstr) */ 168 int remove; /* Bulk to notify the local client of, and remove, next */ 169 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS]; 170 }; 171 172 struct remote_event { 173 int armed; 174 int fired; 175 u32 __unused; 176 }; 177 178 struct opaque_platform_state; 179 180 struct vchiq_slot { 181 char data[VCHIQ_SLOT_SIZE]; 182 }; 183 184 struct vchiq_slot_info { 185 /* Use two counters rather than one to avoid the need for a mutex. */ 186 short use_count; 187 short release_count; 188 }; 189 190 struct vchiq_service { 191 struct vchiq_service_base base; 192 unsigned int handle; 193 struct kref ref_count; 194 struct rcu_head rcu; 195 int srvstate; 196 void (*userdata_term)(void *userdata); 197 unsigned int localport; 198 unsigned int remoteport; 199 int public_fourcc; 200 int client_id; 201 char auto_close; 202 char sync; 203 char closing; 204 char trace; 205 atomic_t poll_flags; 206 short version; 207 short version_min; 208 short peer_version; 209 210 struct vchiq_state *state; 211 struct vchiq_instance *instance; 212 213 int service_use_count; 214 215 struct vchiq_bulk_queue bulk_tx; 216 struct vchiq_bulk_queue bulk_rx; 217 218 struct completion remove_event; 219 struct completion bulk_remove_event; 220 struct mutex bulk_mutex; 221 222 struct service_stats_struct { 223 int quota_stalls; 224 int slot_stalls; 225 int bulk_stalls; 226 int error_count; 227 int ctrl_tx_count; 228 int ctrl_rx_count; 229 int bulk_tx_count; 230 int bulk_rx_count; 231 int bulk_aborted_count; 232 u64 ctrl_tx_bytes; 233 u64 ctrl_rx_bytes; 234 u64 bulk_tx_bytes; 235 u64 bulk_rx_bytes; 236 } stats; 237 238 int msg_queue_read; 239 int msg_queue_write; 240 struct completion msg_queue_pop; 241 struct completion msg_queue_push; 242 struct vchiq_header *msg_queue[VCHIQ_MAX_SLOTS]; 243 }; 244 245 /* 246 * The quota information is outside struct vchiq_service so that it can 247 * be statically allocated, since for accounting reasons a service's slot 248 * usage is carried over between users of the same port number. 249 */ 250 struct vchiq_service_quota { 251 unsigned short slot_quota; 252 unsigned short slot_use_count; 253 unsigned short message_quota; 254 unsigned short message_use_count; 255 struct completion quota_event; 256 int previous_tx_index; 257 }; 258 259 struct vchiq_shared_state { 260 /* A non-zero value here indicates that the content is valid. */ 261 int initialised; 262 263 /* The first and last (inclusive) slots allocated to the owner. */ 264 int slot_first; 265 int slot_last; 266 267 /* The slot allocated to synchronous messages from the owner. */ 268 int slot_sync; 269 270 /* 271 * Signalling this event indicates that owner's slot handler thread 272 * should run. 273 */ 274 struct remote_event trigger; 275 276 /* 277 * Indicates the byte position within the stream where the next message 278 * will be written. The least significant bits are an index into the 279 * slot. The next bits are the index of the slot in slot_queue. 280 */ 281 int tx_pos; 282 283 /* This event should be signalled when a slot is recycled. */ 284 struct remote_event recycle; 285 286 /* The slot_queue index where the next recycled slot will be written. */ 287 int slot_queue_recycle; 288 289 /* This event should be signalled when a synchronous message is sent. */ 290 struct remote_event sync_trigger; 291 292 /* 293 * This event should be signalled when a synchronous message has been 294 * released. 295 */ 296 struct remote_event sync_release; 297 298 /* A circular buffer of slot indexes. */ 299 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE]; 300 301 /* Debugging state */ 302 int debug[DEBUG_MAX]; 303 }; 304 305 struct vchiq_slot_zero { 306 int magic; 307 short version; 308 short version_min; 309 int slot_zero_size; 310 int slot_size; 311 int max_slots; 312 int max_slots_per_side; 313 int platform_data[2]; 314 struct vchiq_shared_state master; 315 struct vchiq_shared_state slave; 316 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS]; 317 }; 318 319 struct vchiq_state { 320 int id; 321 int initialised; 322 enum vchiq_connstate conn_state; 323 short version_common; 324 325 struct vchiq_shared_state *local; 326 struct vchiq_shared_state *remote; 327 struct vchiq_slot *slot_data; 328 329 unsigned short default_slot_quota; 330 unsigned short default_message_quota; 331 332 /* Event indicating connect message received */ 333 struct completion connect; 334 335 /* Mutex protecting services */ 336 struct mutex mutex; 337 struct vchiq_instance **instance; 338 339 /* Processes incoming messages */ 340 struct task_struct *slot_handler_thread; 341 342 /* Processes recycled slots */ 343 struct task_struct *recycle_thread; 344 345 /* Processes synchronous messages */ 346 struct task_struct *sync_thread; 347 348 /* Local implementation of the trigger remote event */ 349 wait_queue_head_t trigger_event; 350 351 /* Local implementation of the recycle remote event */ 352 wait_queue_head_t recycle_event; 353 354 /* Local implementation of the sync trigger remote event */ 355 wait_queue_head_t sync_trigger_event; 356 357 /* Local implementation of the sync release remote event */ 358 wait_queue_head_t sync_release_event; 359 360 char *tx_data; 361 char *rx_data; 362 struct vchiq_slot_info *rx_info; 363 364 struct mutex slot_mutex; 365 366 struct mutex recycle_mutex; 367 368 struct mutex sync_mutex; 369 370 struct mutex bulk_transfer_mutex; 371 372 /* 373 * Indicates the byte position within the stream from where the next 374 * message will be read. The least significant bits are an index into 375 * the slot.The next bits are the index of the slot in 376 * remote->slot_queue. 377 */ 378 int rx_pos; 379 380 /* 381 * A cached copy of local->tx_pos. Only write to local->tx_pos, and read 382 * from remote->tx_pos. 383 */ 384 int local_tx_pos; 385 386 /* The slot_queue index of the slot to become available next. */ 387 int slot_queue_available; 388 389 /* A flag to indicate if any poll has been requested */ 390 int poll_needed; 391 392 /* Ths index of the previous slot used for data messages. */ 393 int previous_data_index; 394 395 /* The number of slots occupied by data messages. */ 396 unsigned short data_use_count; 397 398 /* The maximum number of slots to be occupied by data messages. */ 399 unsigned short data_quota; 400 401 /* An array of bit sets indicating which services must be polled. */ 402 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)]; 403 404 /* The number of the first unused service */ 405 int unused_service; 406 407 /* Signalled when a free slot becomes available. */ 408 struct completion slot_available_event; 409 410 struct completion slot_remove_event; 411 412 /* Signalled when a free data slot becomes available. */ 413 struct completion data_quota_event; 414 415 struct state_stats_struct { 416 int slot_stalls; 417 int data_stalls; 418 int ctrl_tx_count; 419 int ctrl_rx_count; 420 int error_count; 421 } stats; 422 423 struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES]; 424 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES]; 425 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS]; 426 427 struct opaque_platform_state *platform_state; 428 }; 429 430 struct bulk_waiter { 431 struct vchiq_bulk *bulk; 432 struct completion event; 433 int actual; 434 }; 435 436 struct vchiq_config { 437 unsigned int max_msg_size; 438 unsigned int bulk_threshold; /* The message size above which it 439 * is better to use a bulk transfer 440 * (<= max_msg_size) 441 */ 442 unsigned int max_outstanding_bulks; 443 unsigned int max_services; 444 short version; /* The version of VCHIQ */ 445 short version_min; /* The minimum compatible version of VCHIQ */ 446 }; 447 448 extern spinlock_t bulk_waiter_spinlock; 449 450 extern int vchiq_core_log_level; 451 extern int vchiq_core_msg_log_level; 452 extern int vchiq_sync_log_level; 453 454 extern struct vchiq_state *vchiq_states[VCHIQ_MAX_STATES]; 455 456 extern const char * 457 get_conn_state_name(enum vchiq_connstate conn_state); 458 459 extern struct vchiq_slot_zero * 460 vchiq_init_slots(void *mem_base, int mem_size); 461 462 extern int 463 vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero); 464 465 extern enum vchiq_status 466 vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance); 467 468 struct vchiq_service * 469 vchiq_add_service_internal(struct vchiq_state *state, 470 const struct vchiq_service_params_kernel *params, 471 int srvstate, struct vchiq_instance *instance, 472 void (*userdata_term)(void *userdata)); 473 474 extern enum vchiq_status 475 vchiq_open_service_internal(struct vchiq_service *service, int client_id); 476 477 extern enum vchiq_status 478 vchiq_close_service_internal(struct vchiq_service *service, int close_recvd); 479 480 extern void 481 vchiq_terminate_service_internal(struct vchiq_service *service); 482 483 extern void 484 vchiq_free_service_internal(struct vchiq_service *service); 485 486 extern void 487 vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance); 488 489 extern void 490 remote_event_pollall(struct vchiq_state *state); 491 492 extern enum vchiq_status 493 vchiq_bulk_transfer(unsigned int handle, void *offset, void __user *uoffset, 494 int size, void *userdata, enum vchiq_bulk_mode mode, 495 enum vchiq_bulk_dir dir); 496 497 extern int 498 vchiq_dump_state(void *dump_context, struct vchiq_state *state); 499 500 extern int 501 vchiq_dump_service_state(void *dump_context, struct vchiq_service *service); 502 503 extern void 504 vchiq_loud_error_header(void); 505 506 extern void 507 vchiq_loud_error_footer(void); 508 509 extern void 510 request_poll(struct vchiq_state *state, struct vchiq_service *service, 511 int poll_type); 512 513 static inline struct vchiq_service * 514 handle_to_service(unsigned int handle) 515 { 516 int idx = handle & (VCHIQ_MAX_SERVICES - 1); 517 struct vchiq_state *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) & 518 (VCHIQ_MAX_STATES - 1)]; 519 520 if (!state) 521 return NULL; 522 return rcu_dereference(state->services[idx]); 523 } 524 525 extern struct vchiq_service * 526 find_service_by_handle(unsigned int handle); 527 528 extern struct vchiq_service * 529 find_service_by_port(struct vchiq_state *state, int localport); 530 531 extern struct vchiq_service * 532 find_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 533 534 extern struct vchiq_service * 535 find_closed_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 536 537 extern struct vchiq_service * 538 __next_service_by_instance(struct vchiq_state *state, 539 struct vchiq_instance *instance, 540 int *pidx); 541 542 extern struct vchiq_service * 543 next_service_by_instance(struct vchiq_state *state, 544 struct vchiq_instance *instance, 545 int *pidx); 546 547 extern void 548 vchiq_service_get(struct vchiq_service *service); 549 550 extern void 551 vchiq_service_put(struct vchiq_service *service); 552 553 extern enum vchiq_status 554 vchiq_queue_message(unsigned int handle, 555 ssize_t (*copy_callback)(void *context, void *dest, 556 size_t offset, size_t maxsize), 557 void *context, 558 size_t size); 559 560 int vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, void __user *uoffset, 561 int size, int dir); 562 563 void vchiq_complete_bulk(struct vchiq_bulk *bulk); 564 565 void remote_event_signal(struct remote_event *event); 566 567 int vchiq_dump(void *dump_context, const char *str, int len); 568 569 int vchiq_dump_platform_state(void *dump_context); 570 571 int vchiq_dump_platform_instances(void *dump_context); 572 573 int vchiq_dump_platform_service_state(void *dump_context, struct vchiq_service *service); 574 575 int vchiq_use_service_internal(struct vchiq_service *service); 576 577 int vchiq_release_service_internal(struct vchiq_service *service); 578 579 void vchiq_on_remote_use(struct vchiq_state *state); 580 581 void vchiq_on_remote_release(struct vchiq_state *state); 582 583 int vchiq_platform_init_state(struct vchiq_state *state); 584 585 enum vchiq_status vchiq_check_service(struct vchiq_service *service); 586 587 void vchiq_on_remote_use_active(struct vchiq_state *state); 588 589 enum vchiq_status vchiq_send_remote_use(struct vchiq_state *state); 590 591 enum vchiq_status vchiq_send_remote_use_active(struct vchiq_state *state); 592 593 void vchiq_platform_conn_state_changed(struct vchiq_state *state, 594 enum vchiq_connstate oldstate, 595 enum vchiq_connstate newstate); 596 597 void vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate); 598 599 void vchiq_log_dump_mem(const char *label, u32 addr, const void *void_mem, size_t num_bytes); 600 601 enum vchiq_status vchiq_remove_service(unsigned int service); 602 603 int vchiq_get_client_id(unsigned int service); 604 605 void vchiq_get_config(struct vchiq_config *config); 606 607 int vchiq_set_service_option(unsigned int service, enum vchiq_service_option option, int value); 608 609 #endif 610