1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */ 3 4 #ifndef VCHIQ_CORE_H 5 #define VCHIQ_CORE_H 6 7 #include <linux/mutex.h> 8 #include <linux/completion.h> 9 #include <linux/kthread.h> 10 #include <linux/kref.h> 11 #include <linux/rcupdate.h> 12 #include <linux/wait.h> 13 #include <linux/raspberrypi/vchiq.h> 14 15 #include "vchiq_cfg.h" 16 17 /* Do this so that we can test-build the code on non-rpi systems */ 18 #if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) 19 20 #else 21 22 #ifndef dsb 23 #define dsb(a) 24 #endif 25 26 #endif /* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */ 27 28 #define VCHIQ_SERVICE_HANDLE_INVALID 0 29 30 #define VCHIQ_SLOT_SIZE 4096 31 #define VCHIQ_MAX_MSG_SIZE (VCHIQ_SLOT_SIZE - sizeof(struct vchiq_header)) 32 33 /* Run time control of log level, based on KERN_XXX level. */ 34 #define VCHIQ_LOG_DEFAULT 4 35 #define VCHIQ_LOG_ERROR 3 36 #define VCHIQ_LOG_WARNING 4 37 #define VCHIQ_LOG_INFO 6 38 #define VCHIQ_LOG_TRACE 7 39 40 #define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: " 41 42 #ifndef vchiq_log_error 43 #define vchiq_log_error(cat, fmt, ...) \ 44 do { if (cat >= VCHIQ_LOG_ERROR) \ 45 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 46 #endif 47 #ifndef vchiq_log_warning 48 #define vchiq_log_warning(cat, fmt, ...) \ 49 do { if (cat >= VCHIQ_LOG_WARNING) \ 50 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 51 #endif 52 #ifndef vchiq_log_info 53 #define vchiq_log_info(cat, fmt, ...) \ 54 do { if (cat >= VCHIQ_LOG_INFO) \ 55 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 56 #endif 57 #ifndef vchiq_log_trace 58 #define vchiq_log_trace(cat, fmt, ...) \ 59 do { if (cat >= VCHIQ_LOG_TRACE) \ 60 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 61 #endif 62 63 #define vchiq_loud_error(...) \ 64 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__) 65 66 #define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1) 67 #define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1) 68 #define VCHIQ_SLOT_ZERO_SLOTS DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \ 69 VCHIQ_SLOT_SIZE) 70 71 #define VCHIQ_FOURCC_AS_4CHARS(fourcc) \ 72 ((fourcc) >> 24) & 0xff, \ 73 ((fourcc) >> 16) & 0xff, \ 74 ((fourcc) >> 8) & 0xff, \ 75 (fourcc) & 0xff 76 77 #define BITSET_SIZE(b) ((b + 31) >> 5) 78 #define BITSET_WORD(b) (b >> 5) 79 #define BITSET_BIT(b) (1 << (b & 31)) 80 #define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b)) 81 #define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b)) 82 #define BITSET_CLR(bs, b) (bs[BITSET_WORD(b)] &= ~BITSET_BIT(b)) 83 84 enum { 85 DEBUG_ENTRIES, 86 #if VCHIQ_ENABLE_DEBUG 87 DEBUG_SLOT_HANDLER_COUNT, 88 DEBUG_SLOT_HANDLER_LINE, 89 DEBUG_PARSE_LINE, 90 DEBUG_PARSE_HEADER, 91 DEBUG_PARSE_MSGID, 92 DEBUG_AWAIT_COMPLETION_LINE, 93 DEBUG_DEQUEUE_MESSAGE_LINE, 94 DEBUG_SERVICE_CALLBACK_LINE, 95 DEBUG_MSG_QUEUE_FULL_COUNT, 96 DEBUG_COMPLETION_QUEUE_FULL_COUNT, 97 #endif 98 DEBUG_MAX 99 }; 100 101 #if VCHIQ_ENABLE_DEBUG 102 103 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug 104 #define DEBUG_TRACE(d) \ 105 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0) 106 #define DEBUG_VALUE(d, v) \ 107 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0) 108 #define DEBUG_COUNT(d) \ 109 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0) 110 111 #else /* VCHIQ_ENABLE_DEBUG */ 112 113 #define DEBUG_INITIALISE(local) 114 #define DEBUG_TRACE(d) 115 #define DEBUG_VALUE(d, v) 116 #define DEBUG_COUNT(d) 117 118 #endif /* VCHIQ_ENABLE_DEBUG */ 119 120 enum vchiq_connstate { 121 VCHIQ_CONNSTATE_DISCONNECTED, 122 VCHIQ_CONNSTATE_CONNECTING, 123 VCHIQ_CONNSTATE_CONNECTED, 124 VCHIQ_CONNSTATE_PAUSING, 125 VCHIQ_CONNSTATE_PAUSE_SENT, 126 VCHIQ_CONNSTATE_PAUSED, 127 VCHIQ_CONNSTATE_RESUMING, 128 VCHIQ_CONNSTATE_PAUSE_TIMEOUT, 129 VCHIQ_CONNSTATE_RESUME_TIMEOUT 130 }; 131 132 enum { 133 VCHIQ_SRVSTATE_FREE, 134 VCHIQ_SRVSTATE_HIDDEN, 135 VCHIQ_SRVSTATE_LISTENING, 136 VCHIQ_SRVSTATE_OPENING, 137 VCHIQ_SRVSTATE_OPEN, 138 VCHIQ_SRVSTATE_OPENSYNC, 139 VCHIQ_SRVSTATE_CLOSESENT, 140 VCHIQ_SRVSTATE_CLOSERECVD, 141 VCHIQ_SRVSTATE_CLOSEWAIT, 142 VCHIQ_SRVSTATE_CLOSED 143 }; 144 145 enum vchiq_bulk_dir { 146 VCHIQ_BULK_TRANSMIT, 147 VCHIQ_BULK_RECEIVE 148 }; 149 150 struct vchiq_bulk { 151 short mode; 152 short dir; 153 void *userdata; 154 dma_addr_t data; 155 int size; 156 void *remote_data; 157 int remote_size; 158 int actual; 159 }; 160 161 struct vchiq_bulk_queue { 162 int local_insert; /* Where to insert the next local bulk */ 163 int remote_insert; /* Where to insert the next remote bulk (master) */ 164 int process; /* Bulk to transfer next */ 165 int remote_notify; /* Bulk to notify the remote client of next (mstr) */ 166 int remove; /* Bulk to notify the local client of, and remove, next */ 167 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS]; 168 }; 169 170 struct remote_event { 171 int armed; 172 int fired; 173 u32 __unused; 174 }; 175 176 struct opaque_platform_state; 177 178 struct vchiq_slot { 179 char data[VCHIQ_SLOT_SIZE]; 180 }; 181 182 struct vchiq_slot_info { 183 /* Use two counters rather than one to avoid the need for a mutex. */ 184 short use_count; 185 short release_count; 186 }; 187 188 struct vchiq_service { 189 struct vchiq_service_base base; 190 unsigned int handle; 191 struct kref ref_count; 192 struct rcu_head rcu; 193 int srvstate; 194 void (*userdata_term)(void *userdata); 195 unsigned int localport; 196 unsigned int remoteport; 197 int public_fourcc; 198 int client_id; 199 char auto_close; 200 char sync; 201 char closing; 202 char trace; 203 atomic_t poll_flags; 204 short version; 205 short version_min; 206 short peer_version; 207 208 struct vchiq_state *state; 209 struct vchiq_instance *instance; 210 211 int service_use_count; 212 213 struct vchiq_bulk_queue bulk_tx; 214 struct vchiq_bulk_queue bulk_rx; 215 216 struct completion remove_event; 217 struct completion bulk_remove_event; 218 struct mutex bulk_mutex; 219 220 struct service_stats_struct { 221 int quota_stalls; 222 int slot_stalls; 223 int bulk_stalls; 224 int error_count; 225 int ctrl_tx_count; 226 int ctrl_rx_count; 227 int bulk_tx_count; 228 int bulk_rx_count; 229 int bulk_aborted_count; 230 u64 ctrl_tx_bytes; 231 u64 ctrl_rx_bytes; 232 u64 bulk_tx_bytes; 233 u64 bulk_rx_bytes; 234 } stats; 235 236 int msg_queue_read; 237 int msg_queue_write; 238 struct completion msg_queue_pop; 239 struct completion msg_queue_push; 240 struct vchiq_header *msg_queue[VCHIQ_MAX_SLOTS]; 241 }; 242 243 /* 244 * The quota information is outside struct vchiq_service so that it can 245 * be statically allocated, since for accounting reasons a service's slot 246 * usage is carried over between users of the same port number. 247 */ 248 struct vchiq_service_quota { 249 unsigned short slot_quota; 250 unsigned short slot_use_count; 251 unsigned short message_quota; 252 unsigned short message_use_count; 253 struct completion quota_event; 254 int previous_tx_index; 255 }; 256 257 struct vchiq_shared_state { 258 /* A non-zero value here indicates that the content is valid. */ 259 int initialised; 260 261 /* The first and last (inclusive) slots allocated to the owner. */ 262 int slot_first; 263 int slot_last; 264 265 /* The slot allocated to synchronous messages from the owner. */ 266 int slot_sync; 267 268 /* 269 * Signalling this event indicates that owner's slot handler thread 270 * should run. 271 */ 272 struct remote_event trigger; 273 274 /* 275 * Indicates the byte position within the stream where the next message 276 * will be written. The least significant bits are an index into the 277 * slot. The next bits are the index of the slot in slot_queue. 278 */ 279 int tx_pos; 280 281 /* This event should be signalled when a slot is recycled. */ 282 struct remote_event recycle; 283 284 /* The slot_queue index where the next recycled slot will be written. */ 285 int slot_queue_recycle; 286 287 /* This event should be signalled when a synchronous message is sent. */ 288 struct remote_event sync_trigger; 289 290 /* 291 * This event should be signalled when a synchronous message has been 292 * released. 293 */ 294 struct remote_event sync_release; 295 296 /* A circular buffer of slot indexes. */ 297 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE]; 298 299 /* Debugging state */ 300 int debug[DEBUG_MAX]; 301 }; 302 303 struct vchiq_slot_zero { 304 int magic; 305 short version; 306 short version_min; 307 int slot_zero_size; 308 int slot_size; 309 int max_slots; 310 int max_slots_per_side; 311 int platform_data[2]; 312 struct vchiq_shared_state master; 313 struct vchiq_shared_state slave; 314 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS]; 315 }; 316 317 struct vchiq_state { 318 int id; 319 int initialised; 320 enum vchiq_connstate conn_state; 321 short version_common; 322 323 struct vchiq_shared_state *local; 324 struct vchiq_shared_state *remote; 325 struct vchiq_slot *slot_data; 326 327 unsigned short default_slot_quota; 328 unsigned short default_message_quota; 329 330 /* Event indicating connect message received */ 331 struct completion connect; 332 333 /* Mutex protecting services */ 334 struct mutex mutex; 335 struct vchiq_instance **instance; 336 337 /* Processes incoming messages */ 338 struct task_struct *slot_handler_thread; 339 340 /* Processes recycled slots */ 341 struct task_struct *recycle_thread; 342 343 /* Processes synchronous messages */ 344 struct task_struct *sync_thread; 345 346 /* Local implementation of the trigger remote event */ 347 wait_queue_head_t trigger_event; 348 349 /* Local implementation of the recycle remote event */ 350 wait_queue_head_t recycle_event; 351 352 /* Local implementation of the sync trigger remote event */ 353 wait_queue_head_t sync_trigger_event; 354 355 /* Local implementation of the sync release remote event */ 356 wait_queue_head_t sync_release_event; 357 358 char *tx_data; 359 char *rx_data; 360 struct vchiq_slot_info *rx_info; 361 362 struct mutex slot_mutex; 363 364 struct mutex recycle_mutex; 365 366 struct mutex sync_mutex; 367 368 struct mutex bulk_transfer_mutex; 369 370 /* 371 * Indicates the byte position within the stream from where the next 372 * message will be read. The least significant bits are an index into 373 * the slot.The next bits are the index of the slot in 374 * remote->slot_queue. 375 */ 376 int rx_pos; 377 378 /* 379 * A cached copy of local->tx_pos. Only write to local->tx_pos, and read 380 * from remote->tx_pos. 381 */ 382 int local_tx_pos; 383 384 /* The slot_queue index of the slot to become available next. */ 385 int slot_queue_available; 386 387 /* A flag to indicate if any poll has been requested */ 388 int poll_needed; 389 390 /* Ths index of the previous slot used for data messages. */ 391 int previous_data_index; 392 393 /* The number of slots occupied by data messages. */ 394 unsigned short data_use_count; 395 396 /* The maximum number of slots to be occupied by data messages. */ 397 unsigned short data_quota; 398 399 /* An array of bit sets indicating which services must be polled. */ 400 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)]; 401 402 /* The number of the first unused service */ 403 int unused_service; 404 405 /* Signalled when a free slot becomes available. */ 406 struct completion slot_available_event; 407 408 struct completion slot_remove_event; 409 410 /* Signalled when a free data slot becomes available. */ 411 struct completion data_quota_event; 412 413 struct state_stats_struct { 414 int slot_stalls; 415 int data_stalls; 416 int ctrl_tx_count; 417 int ctrl_rx_count; 418 int error_count; 419 } stats; 420 421 struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES]; 422 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES]; 423 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS]; 424 425 struct opaque_platform_state *platform_state; 426 }; 427 428 struct bulk_waiter { 429 struct vchiq_bulk *bulk; 430 struct completion event; 431 int actual; 432 }; 433 434 struct vchiq_config { 435 unsigned int max_msg_size; 436 unsigned int bulk_threshold; /* The message size above which it 437 * is better to use a bulk transfer 438 * (<= max_msg_size) 439 */ 440 unsigned int max_outstanding_bulks; 441 unsigned int max_services; 442 short version; /* The version of VCHIQ */ 443 short version_min; /* The minimum compatible version of VCHIQ */ 444 }; 445 446 extern spinlock_t bulk_waiter_spinlock; 447 448 extern int vchiq_core_log_level; 449 extern int vchiq_core_msg_log_level; 450 extern int vchiq_sync_log_level; 451 452 extern struct vchiq_state *vchiq_states[VCHIQ_MAX_STATES]; 453 454 extern const char * 455 get_conn_state_name(enum vchiq_connstate conn_state); 456 457 extern struct vchiq_slot_zero * 458 vchiq_init_slots(void *mem_base, int mem_size); 459 460 extern int 461 vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero); 462 463 extern enum vchiq_status 464 vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance); 465 466 struct vchiq_service * 467 vchiq_add_service_internal(struct vchiq_state *state, 468 const struct vchiq_service_params_kernel *params, 469 int srvstate, struct vchiq_instance *instance, 470 void (*userdata_term)(void *userdata)); 471 472 extern enum vchiq_status 473 vchiq_open_service_internal(struct vchiq_service *service, int client_id); 474 475 extern enum vchiq_status 476 vchiq_close_service_internal(struct vchiq_service *service, int close_recvd); 477 478 extern void 479 vchiq_terminate_service_internal(struct vchiq_service *service); 480 481 extern void 482 vchiq_free_service_internal(struct vchiq_service *service); 483 484 extern void 485 vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance); 486 487 extern void 488 remote_event_pollall(struct vchiq_state *state); 489 490 extern enum vchiq_status 491 vchiq_bulk_transfer(unsigned int handle, void *offset, void __user *uoffset, 492 int size, void *userdata, enum vchiq_bulk_mode mode, 493 enum vchiq_bulk_dir dir); 494 495 extern int 496 vchiq_dump_state(void *dump_context, struct vchiq_state *state); 497 498 extern int 499 vchiq_dump_service_state(void *dump_context, struct vchiq_service *service); 500 501 extern void 502 vchiq_loud_error_header(void); 503 504 extern void 505 vchiq_loud_error_footer(void); 506 507 extern void 508 request_poll(struct vchiq_state *state, struct vchiq_service *service, 509 int poll_type); 510 511 static inline struct vchiq_service * 512 handle_to_service(unsigned int handle) 513 { 514 int idx = handle & (VCHIQ_MAX_SERVICES - 1); 515 struct vchiq_state *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) & 516 (VCHIQ_MAX_STATES - 1)]; 517 518 if (!state) 519 return NULL; 520 return rcu_dereference(state->services[idx]); 521 } 522 523 extern struct vchiq_service * 524 find_service_by_handle(unsigned int handle); 525 526 extern struct vchiq_service * 527 find_service_by_port(struct vchiq_state *state, int localport); 528 529 extern struct vchiq_service * 530 find_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 531 532 extern struct vchiq_service * 533 find_closed_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 534 535 extern struct vchiq_service * 536 __next_service_by_instance(struct vchiq_state *state, 537 struct vchiq_instance *instance, 538 int *pidx); 539 540 extern struct vchiq_service * 541 next_service_by_instance(struct vchiq_state *state, 542 struct vchiq_instance *instance, 543 int *pidx); 544 545 extern void 546 vchiq_service_get(struct vchiq_service *service); 547 548 extern void 549 vchiq_service_put(struct vchiq_service *service); 550 551 extern enum vchiq_status 552 vchiq_queue_message(unsigned int handle, 553 ssize_t (*copy_callback)(void *context, void *dest, 554 size_t offset, size_t maxsize), 555 void *context, 556 size_t size); 557 558 int vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, void __user *uoffset, 559 int size, int dir); 560 561 void vchiq_complete_bulk(struct vchiq_bulk *bulk); 562 563 void remote_event_signal(struct remote_event *event); 564 565 int vchiq_dump(void *dump_context, const char *str, int len); 566 567 int vchiq_dump_platform_state(void *dump_context); 568 569 int vchiq_dump_platform_instances(void *dump_context); 570 571 int vchiq_dump_platform_service_state(void *dump_context, struct vchiq_service *service); 572 573 int vchiq_use_service_internal(struct vchiq_service *service); 574 575 int vchiq_release_service_internal(struct vchiq_service *service); 576 577 void vchiq_on_remote_use(struct vchiq_state *state); 578 579 void vchiq_on_remote_release(struct vchiq_state *state); 580 581 int vchiq_platform_init_state(struct vchiq_state *state); 582 583 enum vchiq_status vchiq_check_service(struct vchiq_service *service); 584 585 void vchiq_on_remote_use_active(struct vchiq_state *state); 586 587 enum vchiq_status vchiq_send_remote_use(struct vchiq_state *state); 588 589 enum vchiq_status vchiq_send_remote_use_active(struct vchiq_state *state); 590 591 void vchiq_platform_conn_state_changed(struct vchiq_state *state, 592 enum vchiq_connstate oldstate, 593 enum vchiq_connstate newstate); 594 595 void vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate); 596 597 void vchiq_log_dump_mem(const char *label, u32 addr, const void *void_mem, size_t num_bytes); 598 599 enum vchiq_status vchiq_remove_service(unsigned int service); 600 601 int vchiq_get_client_id(unsigned int service); 602 603 void vchiq_get_config(struct vchiq_config *config); 604 605 int vchiq_set_service_option(unsigned int service, enum vchiq_service_option option, int value); 606 607 #endif 608