1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */ 3 4 #ifndef VCHIQ_CORE_H 5 #define VCHIQ_CORE_H 6 7 #include <linux/mutex.h> 8 #include <linux/completion.h> 9 #include <linux/kthread.h> 10 #include <linux/kref.h> 11 #include <linux/rcupdate.h> 12 #include <linux/wait.h> 13 #include <linux/raspberrypi/vchiq.h> 14 15 #include "vchiq_cfg.h" 16 17 /* Do this so that we can test-build the code on non-rpi systems */ 18 #if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) 19 20 #else 21 22 #ifndef dsb 23 #define dsb(a) 24 #endif 25 26 #endif /* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */ 27 28 #define VCHIQ_SERVICE_HANDLE_INVALID 0 29 30 #define VCHIQ_SLOT_SIZE 4096 31 #define VCHIQ_MAX_MSG_SIZE (VCHIQ_SLOT_SIZE - sizeof(struct vchiq_header)) 32 33 /* Run time control of log level, based on KERN_XXX level. */ 34 #define VCHIQ_LOG_DEFAULT 4 35 #define VCHIQ_LOG_ERROR 3 36 #define VCHIQ_LOG_WARNING 4 37 #define VCHIQ_LOG_INFO 6 38 #define VCHIQ_LOG_TRACE 7 39 40 #define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: " 41 42 #ifndef vchiq_log_error 43 #define vchiq_log_error(cat, fmt, ...) \ 44 do { if (cat >= VCHIQ_LOG_ERROR) \ 45 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 46 #endif 47 #ifndef vchiq_log_warning 48 #define vchiq_log_warning(cat, fmt, ...) \ 49 do { if (cat >= VCHIQ_LOG_WARNING) \ 50 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 51 #endif 52 #ifndef vchiq_log_info 53 #define vchiq_log_info(cat, fmt, ...) \ 54 do { if (cat >= VCHIQ_LOG_INFO) \ 55 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 56 #endif 57 #ifndef vchiq_log_trace 58 #define vchiq_log_trace(cat, fmt, ...) \ 59 do { if (cat >= VCHIQ_LOG_TRACE) \ 60 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0) 61 #endif 62 63 #define vchiq_loud_error(...) \ 64 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__) 65 66 #define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1) 67 #define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1) 68 #define VCHIQ_SLOT_ZERO_SLOTS DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \ 69 VCHIQ_SLOT_SIZE) 70 71 #define VCHIQ_FOURCC_AS_4CHARS(fourcc) \ 72 ((fourcc) >> 24) & 0xff, \ 73 ((fourcc) >> 16) & 0xff, \ 74 ((fourcc) >> 8) & 0xff, \ 75 (fourcc) & 0xff 76 77 #define BITSET_SIZE(b) ((b + 31) >> 5) 78 #define BITSET_WORD(b) (b >> 5) 79 #define BITSET_BIT(b) (1 << (b & 31)) 80 #define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b)) 81 #define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b)) 82 83 enum { 84 DEBUG_ENTRIES, 85 #if VCHIQ_ENABLE_DEBUG 86 DEBUG_SLOT_HANDLER_COUNT, 87 DEBUG_SLOT_HANDLER_LINE, 88 DEBUG_PARSE_LINE, 89 DEBUG_PARSE_HEADER, 90 DEBUG_PARSE_MSGID, 91 DEBUG_AWAIT_COMPLETION_LINE, 92 DEBUG_DEQUEUE_MESSAGE_LINE, 93 DEBUG_SERVICE_CALLBACK_LINE, 94 DEBUG_MSG_QUEUE_FULL_COUNT, 95 DEBUG_COMPLETION_QUEUE_FULL_COUNT, 96 #endif 97 DEBUG_MAX 98 }; 99 100 #if VCHIQ_ENABLE_DEBUG 101 102 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug 103 #define DEBUG_TRACE(d) \ 104 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0) 105 #define DEBUG_VALUE(d, v) \ 106 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0) 107 #define DEBUG_COUNT(d) \ 108 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0) 109 110 #else /* VCHIQ_ENABLE_DEBUG */ 111 112 #define DEBUG_INITIALISE(local) 113 #define DEBUG_TRACE(d) 114 #define DEBUG_VALUE(d, v) 115 #define DEBUG_COUNT(d) 116 117 #endif /* VCHIQ_ENABLE_DEBUG */ 118 119 enum vchiq_connstate { 120 VCHIQ_CONNSTATE_DISCONNECTED, 121 VCHIQ_CONNSTATE_CONNECTING, 122 VCHIQ_CONNSTATE_CONNECTED, 123 VCHIQ_CONNSTATE_PAUSING, 124 VCHIQ_CONNSTATE_PAUSE_SENT, 125 VCHIQ_CONNSTATE_PAUSED, 126 VCHIQ_CONNSTATE_RESUMING, 127 VCHIQ_CONNSTATE_PAUSE_TIMEOUT, 128 VCHIQ_CONNSTATE_RESUME_TIMEOUT 129 }; 130 131 enum { 132 VCHIQ_SRVSTATE_FREE, 133 VCHIQ_SRVSTATE_HIDDEN, 134 VCHIQ_SRVSTATE_LISTENING, 135 VCHIQ_SRVSTATE_OPENING, 136 VCHIQ_SRVSTATE_OPEN, 137 VCHIQ_SRVSTATE_OPENSYNC, 138 VCHIQ_SRVSTATE_CLOSESENT, 139 VCHIQ_SRVSTATE_CLOSERECVD, 140 VCHIQ_SRVSTATE_CLOSEWAIT, 141 VCHIQ_SRVSTATE_CLOSED 142 }; 143 144 enum vchiq_bulk_dir { 145 VCHIQ_BULK_TRANSMIT, 146 VCHIQ_BULK_RECEIVE 147 }; 148 149 struct vchiq_bulk { 150 short mode; 151 short dir; 152 void *userdata; 153 dma_addr_t data; 154 int size; 155 void *remote_data; 156 int remote_size; 157 int actual; 158 }; 159 160 struct vchiq_bulk_queue { 161 int local_insert; /* Where to insert the next local bulk */ 162 int remote_insert; /* Where to insert the next remote bulk (master) */ 163 int process; /* Bulk to transfer next */ 164 int remote_notify; /* Bulk to notify the remote client of next (mstr) */ 165 int remove; /* Bulk to notify the local client of, and remove, next */ 166 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS]; 167 }; 168 169 struct remote_event { 170 int armed; 171 int fired; 172 u32 __unused; 173 }; 174 175 struct opaque_platform_state; 176 177 struct vchiq_slot { 178 char data[VCHIQ_SLOT_SIZE]; 179 }; 180 181 struct vchiq_slot_info { 182 /* Use two counters rather than one to avoid the need for a mutex. */ 183 short use_count; 184 short release_count; 185 }; 186 187 struct vchiq_service { 188 struct vchiq_service_base base; 189 unsigned int handle; 190 struct kref ref_count; 191 struct rcu_head rcu; 192 int srvstate; 193 void (*userdata_term)(void *userdata); 194 unsigned int localport; 195 unsigned int remoteport; 196 int public_fourcc; 197 int client_id; 198 char auto_close; 199 char sync; 200 char closing; 201 char trace; 202 atomic_t poll_flags; 203 short version; 204 short version_min; 205 short peer_version; 206 207 struct vchiq_state *state; 208 struct vchiq_instance *instance; 209 210 int service_use_count; 211 212 struct vchiq_bulk_queue bulk_tx; 213 struct vchiq_bulk_queue bulk_rx; 214 215 struct completion remove_event; 216 struct completion bulk_remove_event; 217 struct mutex bulk_mutex; 218 219 struct service_stats_struct { 220 int quota_stalls; 221 int slot_stalls; 222 int bulk_stalls; 223 int error_count; 224 int ctrl_tx_count; 225 int ctrl_rx_count; 226 int bulk_tx_count; 227 int bulk_rx_count; 228 int bulk_aborted_count; 229 u64 ctrl_tx_bytes; 230 u64 ctrl_rx_bytes; 231 u64 bulk_tx_bytes; 232 u64 bulk_rx_bytes; 233 } stats; 234 235 int msg_queue_read; 236 int msg_queue_write; 237 struct completion msg_queue_pop; 238 struct completion msg_queue_push; 239 struct vchiq_header *msg_queue[VCHIQ_MAX_SLOTS]; 240 }; 241 242 /* 243 * The quota information is outside struct vchiq_service so that it can 244 * be statically allocated, since for accounting reasons a service's slot 245 * usage is carried over between users of the same port number. 246 */ 247 struct vchiq_service_quota { 248 unsigned short slot_quota; 249 unsigned short slot_use_count; 250 unsigned short message_quota; 251 unsigned short message_use_count; 252 struct completion quota_event; 253 int previous_tx_index; 254 }; 255 256 struct vchiq_shared_state { 257 /* A non-zero value here indicates that the content is valid. */ 258 int initialised; 259 260 /* The first and last (inclusive) slots allocated to the owner. */ 261 int slot_first; 262 int slot_last; 263 264 /* The slot allocated to synchronous messages from the owner. */ 265 int slot_sync; 266 267 /* 268 * Signalling this event indicates that owner's slot handler thread 269 * should run. 270 */ 271 struct remote_event trigger; 272 273 /* 274 * Indicates the byte position within the stream where the next message 275 * will be written. The least significant bits are an index into the 276 * slot. The next bits are the index of the slot in slot_queue. 277 */ 278 int tx_pos; 279 280 /* This event should be signalled when a slot is recycled. */ 281 struct remote_event recycle; 282 283 /* The slot_queue index where the next recycled slot will be written. */ 284 int slot_queue_recycle; 285 286 /* This event should be signalled when a synchronous message is sent. */ 287 struct remote_event sync_trigger; 288 289 /* 290 * This event should be signalled when a synchronous message has been 291 * released. 292 */ 293 struct remote_event sync_release; 294 295 /* A circular buffer of slot indexes. */ 296 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE]; 297 298 /* Debugging state */ 299 int debug[DEBUG_MAX]; 300 }; 301 302 struct vchiq_slot_zero { 303 int magic; 304 short version; 305 short version_min; 306 int slot_zero_size; 307 int slot_size; 308 int max_slots; 309 int max_slots_per_side; 310 int platform_data[2]; 311 struct vchiq_shared_state master; 312 struct vchiq_shared_state slave; 313 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS]; 314 }; 315 316 struct vchiq_state { 317 int id; 318 int initialised; 319 enum vchiq_connstate conn_state; 320 short version_common; 321 322 struct vchiq_shared_state *local; 323 struct vchiq_shared_state *remote; 324 struct vchiq_slot *slot_data; 325 326 unsigned short default_slot_quota; 327 unsigned short default_message_quota; 328 329 /* Event indicating connect message received */ 330 struct completion connect; 331 332 /* Mutex protecting services */ 333 struct mutex mutex; 334 struct vchiq_instance **instance; 335 336 /* Processes incoming messages */ 337 struct task_struct *slot_handler_thread; 338 339 /* Processes recycled slots */ 340 struct task_struct *recycle_thread; 341 342 /* Processes synchronous messages */ 343 struct task_struct *sync_thread; 344 345 /* Local implementation of the trigger remote event */ 346 wait_queue_head_t trigger_event; 347 348 /* Local implementation of the recycle remote event */ 349 wait_queue_head_t recycle_event; 350 351 /* Local implementation of the sync trigger remote event */ 352 wait_queue_head_t sync_trigger_event; 353 354 /* Local implementation of the sync release remote event */ 355 wait_queue_head_t sync_release_event; 356 357 char *tx_data; 358 char *rx_data; 359 struct vchiq_slot_info *rx_info; 360 361 struct mutex slot_mutex; 362 363 struct mutex recycle_mutex; 364 365 struct mutex sync_mutex; 366 367 struct mutex bulk_transfer_mutex; 368 369 /* 370 * Indicates the byte position within the stream from where the next 371 * message will be read. The least significant bits are an index into 372 * the slot.The next bits are the index of the slot in 373 * remote->slot_queue. 374 */ 375 int rx_pos; 376 377 /* 378 * A cached copy of local->tx_pos. Only write to local->tx_pos, and read 379 * from remote->tx_pos. 380 */ 381 int local_tx_pos; 382 383 /* The slot_queue index of the slot to become available next. */ 384 int slot_queue_available; 385 386 /* A flag to indicate if any poll has been requested */ 387 int poll_needed; 388 389 /* Ths index of the previous slot used for data messages. */ 390 int previous_data_index; 391 392 /* The number of slots occupied by data messages. */ 393 unsigned short data_use_count; 394 395 /* The maximum number of slots to be occupied by data messages. */ 396 unsigned short data_quota; 397 398 /* An array of bit sets indicating which services must be polled. */ 399 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)]; 400 401 /* The number of the first unused service */ 402 int unused_service; 403 404 /* Signalled when a free slot becomes available. */ 405 struct completion slot_available_event; 406 407 struct completion slot_remove_event; 408 409 /* Signalled when a free data slot becomes available. */ 410 struct completion data_quota_event; 411 412 struct state_stats_struct { 413 int slot_stalls; 414 int data_stalls; 415 int ctrl_tx_count; 416 int ctrl_rx_count; 417 int error_count; 418 } stats; 419 420 struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES]; 421 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES]; 422 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS]; 423 424 struct opaque_platform_state *platform_state; 425 }; 426 427 struct bulk_waiter { 428 struct vchiq_bulk *bulk; 429 struct completion event; 430 int actual; 431 }; 432 433 struct vchiq_config { 434 unsigned int max_msg_size; 435 unsigned int bulk_threshold; /* The message size above which it 436 * is better to use a bulk transfer 437 * (<= max_msg_size) 438 */ 439 unsigned int max_outstanding_bulks; 440 unsigned int max_services; 441 short version; /* The version of VCHIQ */ 442 short version_min; /* The minimum compatible version of VCHIQ */ 443 }; 444 445 extern spinlock_t bulk_waiter_spinlock; 446 447 extern int vchiq_core_log_level; 448 extern int vchiq_core_msg_log_level; 449 extern int vchiq_sync_log_level; 450 451 extern struct vchiq_state *vchiq_states[VCHIQ_MAX_STATES]; 452 453 extern const char * 454 get_conn_state_name(enum vchiq_connstate conn_state); 455 456 extern struct vchiq_slot_zero * 457 vchiq_init_slots(void *mem_base, int mem_size); 458 459 extern int 460 vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero); 461 462 extern enum vchiq_status 463 vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance); 464 465 struct vchiq_service * 466 vchiq_add_service_internal(struct vchiq_state *state, 467 const struct vchiq_service_params_kernel *params, 468 int srvstate, struct vchiq_instance *instance, 469 void (*userdata_term)(void *userdata)); 470 471 extern enum vchiq_status 472 vchiq_open_service_internal(struct vchiq_service *service, int client_id); 473 474 extern enum vchiq_status 475 vchiq_close_service_internal(struct vchiq_service *service, int close_recvd); 476 477 extern void 478 vchiq_terminate_service_internal(struct vchiq_service *service); 479 480 extern void 481 vchiq_free_service_internal(struct vchiq_service *service); 482 483 extern void 484 vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance); 485 486 extern void 487 remote_event_pollall(struct vchiq_state *state); 488 489 extern enum vchiq_status 490 vchiq_bulk_transfer(unsigned int handle, void *offset, void __user *uoffset, 491 int size, void *userdata, enum vchiq_bulk_mode mode, 492 enum vchiq_bulk_dir dir); 493 494 extern int 495 vchiq_dump_state(void *dump_context, struct vchiq_state *state); 496 497 extern int 498 vchiq_dump_service_state(void *dump_context, struct vchiq_service *service); 499 500 extern void 501 vchiq_loud_error_header(void); 502 503 extern void 504 vchiq_loud_error_footer(void); 505 506 extern void 507 request_poll(struct vchiq_state *state, struct vchiq_service *service, 508 int poll_type); 509 510 static inline struct vchiq_service * 511 handle_to_service(unsigned int handle) 512 { 513 int idx = handle & (VCHIQ_MAX_SERVICES - 1); 514 struct vchiq_state *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) & 515 (VCHIQ_MAX_STATES - 1)]; 516 517 if (!state) 518 return NULL; 519 return rcu_dereference(state->services[idx]); 520 } 521 522 extern struct vchiq_service * 523 find_service_by_handle(unsigned int handle); 524 525 extern struct vchiq_service * 526 find_service_by_port(struct vchiq_state *state, unsigned int localport); 527 528 extern struct vchiq_service * 529 find_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 530 531 extern struct vchiq_service * 532 find_closed_service_for_instance(struct vchiq_instance *instance, unsigned int handle); 533 534 extern struct vchiq_service * 535 __next_service_by_instance(struct vchiq_state *state, 536 struct vchiq_instance *instance, 537 int *pidx); 538 539 extern struct vchiq_service * 540 next_service_by_instance(struct vchiq_state *state, 541 struct vchiq_instance *instance, 542 int *pidx); 543 544 extern void 545 vchiq_service_get(struct vchiq_service *service); 546 547 extern void 548 vchiq_service_put(struct vchiq_service *service); 549 550 extern enum vchiq_status 551 vchiq_queue_message(unsigned int handle, 552 ssize_t (*copy_callback)(void *context, void *dest, 553 size_t offset, size_t maxsize), 554 void *context, 555 size_t size); 556 557 int vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, void __user *uoffset, 558 int size, int dir); 559 560 void vchiq_complete_bulk(struct vchiq_bulk *bulk); 561 562 void remote_event_signal(struct remote_event *event); 563 564 int vchiq_dump(void *dump_context, const char *str, int len); 565 566 int vchiq_dump_platform_state(void *dump_context); 567 568 int vchiq_dump_platform_instances(void *dump_context); 569 570 int vchiq_dump_platform_service_state(void *dump_context, struct vchiq_service *service); 571 572 int vchiq_use_service_internal(struct vchiq_service *service); 573 574 int vchiq_release_service_internal(struct vchiq_service *service); 575 576 void vchiq_on_remote_use(struct vchiq_state *state); 577 578 void vchiq_on_remote_release(struct vchiq_state *state); 579 580 int vchiq_platform_init_state(struct vchiq_state *state); 581 582 enum vchiq_status vchiq_check_service(struct vchiq_service *service); 583 584 void vchiq_on_remote_use_active(struct vchiq_state *state); 585 586 enum vchiq_status vchiq_send_remote_use(struct vchiq_state *state); 587 588 enum vchiq_status vchiq_send_remote_use_active(struct vchiq_state *state); 589 590 void vchiq_platform_conn_state_changed(struct vchiq_state *state, 591 enum vchiq_connstate oldstate, 592 enum vchiq_connstate newstate); 593 594 void vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate); 595 596 void vchiq_log_dump_mem(const char *label, u32 addr, const void *void_mem, size_t num_bytes); 597 598 enum vchiq_status vchiq_remove_service(unsigned int service); 599 600 int vchiq_get_client_id(unsigned int service); 601 602 void vchiq_get_config(struct vchiq_config *config); 603 604 int vchiq_set_service_option(unsigned int service, enum vchiq_service_option option, int value); 605 606 #endif 607