xref: /openbmc/linux/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.h (revision 9a87ffc99ec8eb8d35eed7c4f816d75f5cc9662e)
1  /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2  /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */
3  
4  #ifndef VCHIQ_CORE_H
5  #define VCHIQ_CORE_H
6  
7  #include <linux/mutex.h>
8  #include <linux/completion.h>
9  #include <linux/kthread.h>
10  #include <linux/kref.h>
11  #include <linux/rcupdate.h>
12  #include <linux/wait.h>
13  
14  #include "../../include/linux/raspberrypi/vchiq.h"
15  #include "vchiq_cfg.h"
16  
17  /* Do this so that we can test-build the code on non-rpi systems */
18  #if IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE)
19  
20  #else
21  
22  #ifndef dsb
23  #define dsb(a)
24  #endif
25  
26  #endif	/* IS_ENABLED(CONFIG_RASPBERRYPI_FIRMWARE) */
27  
28  #define VCHIQ_SERVICE_HANDLE_INVALID 0
29  
30  #define VCHIQ_SLOT_SIZE     4096
31  #define VCHIQ_MAX_MSG_SIZE  (VCHIQ_SLOT_SIZE - sizeof(struct vchiq_header))
32  
33  /* Run time control of log level, based on KERN_XXX level. */
34  #define VCHIQ_LOG_DEFAULT  4
35  #define VCHIQ_LOG_ERROR    3
36  #define VCHIQ_LOG_WARNING  4
37  #define VCHIQ_LOG_INFO     6
38  #define VCHIQ_LOG_TRACE    7
39  
40  #define VCHIQ_LOG_PREFIX   KERN_INFO "vchiq: "
41  
42  #ifndef vchiq_log_error
43  #define vchiq_log_error(cat, fmt, ...) \
44  	do { if (cat >= VCHIQ_LOG_ERROR) \
45  		printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
46  #endif
47  #ifndef vchiq_log_warning
48  #define vchiq_log_warning(cat, fmt, ...) \
49  	do { if (cat >= VCHIQ_LOG_WARNING) \
50  		 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
51  #endif
52  #ifndef vchiq_log_info
53  #define vchiq_log_info(cat, fmt, ...) \
54  	do { if (cat >= VCHIQ_LOG_INFO) \
55  		printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
56  #endif
57  #ifndef vchiq_log_trace
58  #define vchiq_log_trace(cat, fmt, ...) \
59  	do { if (cat >= VCHIQ_LOG_TRACE) \
60  		printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
61  #endif
62  
63  #define vchiq_loud_error(...) \
64  	vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__)
65  
66  #define VCHIQ_SLOT_MASK        (VCHIQ_SLOT_SIZE - 1)
67  #define VCHIQ_SLOT_QUEUE_MASK  (VCHIQ_MAX_SLOTS_PER_SIDE - 1)
68  #define VCHIQ_SLOT_ZERO_SLOTS  DIV_ROUND_UP(sizeof(struct vchiq_slot_zero), \
69  					    VCHIQ_SLOT_SIZE)
70  
71  #define VCHIQ_FOURCC_AS_4CHARS(fourcc)	\
72  	((fourcc) >> 24) & 0xff, \
73  	((fourcc) >> 16) & 0xff, \
74  	((fourcc) >>  8) & 0xff, \
75  	(fourcc) & 0xff
76  
77  #define BITSET_SIZE(b)        ((b + 31) >> 5)
78  #define BITSET_WORD(b)        (b >> 5)
79  #define BITSET_BIT(b)         (1 << (b & 31))
80  #define BITSET_IS_SET(bs, b)  (bs[BITSET_WORD(b)] & BITSET_BIT(b))
81  #define BITSET_SET(bs, b)     (bs[BITSET_WORD(b)] |= BITSET_BIT(b))
82  
83  enum {
84  	DEBUG_ENTRIES,
85  #if VCHIQ_ENABLE_DEBUG
86  	DEBUG_SLOT_HANDLER_COUNT,
87  	DEBUG_SLOT_HANDLER_LINE,
88  	DEBUG_PARSE_LINE,
89  	DEBUG_PARSE_HEADER,
90  	DEBUG_PARSE_MSGID,
91  	DEBUG_AWAIT_COMPLETION_LINE,
92  	DEBUG_DEQUEUE_MESSAGE_LINE,
93  	DEBUG_SERVICE_CALLBACK_LINE,
94  	DEBUG_MSG_QUEUE_FULL_COUNT,
95  	DEBUG_COMPLETION_QUEUE_FULL_COUNT,
96  #endif
97  	DEBUG_MAX
98  };
99  
100  #if VCHIQ_ENABLE_DEBUG
101  
102  #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug
103  #define DEBUG_TRACE(d) \
104  	do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0)
105  #define DEBUG_VALUE(d, v) \
106  	do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0)
107  #define DEBUG_COUNT(d) \
108  	do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0)
109  
110  #else /* VCHIQ_ENABLE_DEBUG */
111  
112  #define DEBUG_INITIALISE(local)
113  #define DEBUG_TRACE(d)
114  #define DEBUG_VALUE(d, v)
115  #define DEBUG_COUNT(d)
116  
117  #endif /* VCHIQ_ENABLE_DEBUG */
118  
119  enum vchiq_connstate {
120  	VCHIQ_CONNSTATE_DISCONNECTED,
121  	VCHIQ_CONNSTATE_CONNECTING,
122  	VCHIQ_CONNSTATE_CONNECTED,
123  	VCHIQ_CONNSTATE_PAUSING,
124  	VCHIQ_CONNSTATE_PAUSE_SENT,
125  	VCHIQ_CONNSTATE_PAUSED,
126  	VCHIQ_CONNSTATE_RESUMING,
127  	VCHIQ_CONNSTATE_PAUSE_TIMEOUT,
128  	VCHIQ_CONNSTATE_RESUME_TIMEOUT
129  };
130  
131  enum {
132  	VCHIQ_SRVSTATE_FREE,
133  	VCHIQ_SRVSTATE_HIDDEN,
134  	VCHIQ_SRVSTATE_LISTENING,
135  	VCHIQ_SRVSTATE_OPENING,
136  	VCHIQ_SRVSTATE_OPEN,
137  	VCHIQ_SRVSTATE_OPENSYNC,
138  	VCHIQ_SRVSTATE_CLOSESENT,
139  	VCHIQ_SRVSTATE_CLOSERECVD,
140  	VCHIQ_SRVSTATE_CLOSEWAIT,
141  	VCHIQ_SRVSTATE_CLOSED
142  };
143  
144  enum vchiq_bulk_dir {
145  	VCHIQ_BULK_TRANSMIT,
146  	VCHIQ_BULK_RECEIVE
147  };
148  
149  struct vchiq_bulk {
150  	short mode;
151  	short dir;
152  	void *userdata;
153  	dma_addr_t data;
154  	int size;
155  	void *remote_data;
156  	int remote_size;
157  	int actual;
158  };
159  
160  struct vchiq_bulk_queue {
161  	int local_insert;  /* Where to insert the next local bulk */
162  	int remote_insert; /* Where to insert the next remote bulk (master) */
163  	int process;       /* Bulk to transfer next */
164  	int remote_notify; /* Bulk to notify the remote client of next (mstr) */
165  	int remove;        /* Bulk to notify the local client of, and remove, next */
166  	struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS];
167  };
168  
169  /*
170   * Remote events provide a way of presenting several virtual doorbells to a
171   * peer (ARM host to VPU) using only one physical doorbell. They can be thought
172   * of as a way for the peer to signal a semaphore, in this case implemented as
173   * a workqueue.
174   *
175   * Remote events remain signalled until acknowledged by the receiver, and they
176   * are non-counting. They are designed in such a way as to minimise the number
177   * of interrupts and avoid unnecessary waiting.
178   *
179   * A remote_event is as small data structures that live in shared memory. It
180   * comprises two booleans - armed and fired:
181   *
182   * The sender sets fired when they signal the receiver.
183   * If fired is set, the receiver has been signalled and need not wait.
184   * The receiver sets the armed field before they begin to wait.
185   * If armed is set, the receiver is waiting and wishes to be woken by interrupt.
186   */
187  struct remote_event {
188  	int armed;
189  	int fired;
190  	u32 __unused;
191  };
192  
193  struct opaque_platform_state;
194  
195  struct vchiq_slot {
196  	char data[VCHIQ_SLOT_SIZE];
197  };
198  
199  struct vchiq_slot_info {
200  	/* Use two counters rather than one to avoid the need for a mutex. */
201  	short use_count;
202  	short release_count;
203  };
204  
205  struct vchiq_service {
206  	struct vchiq_service_base base;
207  	unsigned int handle;
208  	struct kref ref_count;
209  	struct rcu_head rcu;
210  	int srvstate;
211  	void (*userdata_term)(void *userdata);
212  	unsigned int localport;
213  	unsigned int remoteport;
214  	int public_fourcc;
215  	int client_id;
216  	char auto_close;
217  	char sync;
218  	char closing;
219  	char trace;
220  	atomic_t poll_flags;
221  	short version;
222  	short version_min;
223  	short peer_version;
224  
225  	struct vchiq_state *state;
226  	struct vchiq_instance *instance;
227  
228  	int service_use_count;
229  
230  	struct vchiq_bulk_queue bulk_tx;
231  	struct vchiq_bulk_queue bulk_rx;
232  
233  	struct completion remove_event;
234  	struct completion bulk_remove_event;
235  	struct mutex bulk_mutex;
236  
237  	struct service_stats_struct {
238  		int quota_stalls;
239  		int slot_stalls;
240  		int bulk_stalls;
241  		int error_count;
242  		int ctrl_tx_count;
243  		int ctrl_rx_count;
244  		int bulk_tx_count;
245  		int bulk_rx_count;
246  		int bulk_aborted_count;
247  		u64 ctrl_tx_bytes;
248  		u64 ctrl_rx_bytes;
249  		u64 bulk_tx_bytes;
250  		u64 bulk_rx_bytes;
251  	} stats;
252  
253  	int msg_queue_read;
254  	int msg_queue_write;
255  	struct completion msg_queue_pop;
256  	struct completion msg_queue_push;
257  	struct vchiq_header *msg_queue[VCHIQ_MAX_SLOTS];
258  };
259  
260  /*
261   * The quota information is outside struct vchiq_service so that it can
262   * be statically allocated, since for accounting reasons a service's slot
263   * usage is carried over between users of the same port number.
264   */
265  struct vchiq_service_quota {
266  	unsigned short slot_quota;
267  	unsigned short slot_use_count;
268  	unsigned short message_quota;
269  	unsigned short message_use_count;
270  	struct completion quota_event;
271  	int previous_tx_index;
272  };
273  
274  struct vchiq_shared_state {
275  	/* A non-zero value here indicates that the content is valid. */
276  	int initialised;
277  
278  	/* The first and last (inclusive) slots allocated to the owner. */
279  	int slot_first;
280  	int slot_last;
281  
282  	/* The slot allocated to synchronous messages from the owner. */
283  	int slot_sync;
284  
285  	/*
286  	 * Signalling this event indicates that owner's slot handler thread
287  	 * should run.
288  	 */
289  	struct remote_event trigger;
290  
291  	/*
292  	 * Indicates the byte position within the stream where the next message
293  	 * will be written. The least significant bits are an index into the
294  	 * slot. The next bits are the index of the slot in slot_queue.
295  	 */
296  	int tx_pos;
297  
298  	/* This event should be signalled when a slot is recycled. */
299  	struct remote_event recycle;
300  
301  	/* The slot_queue index where the next recycled slot will be written. */
302  	int slot_queue_recycle;
303  
304  	/* This event should be signalled when a synchronous message is sent. */
305  	struct remote_event sync_trigger;
306  
307  	/*
308  	 * This event should be signalled when a synchronous message has been
309  	 * released.
310  	 */
311  	struct remote_event sync_release;
312  
313  	/* A circular buffer of slot indexes. */
314  	int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE];
315  
316  	/* Debugging state */
317  	int debug[DEBUG_MAX];
318  };
319  
320  struct vchiq_slot_zero {
321  	int magic;
322  	short version;
323  	short version_min;
324  	int slot_zero_size;
325  	int slot_size;
326  	int max_slots;
327  	int max_slots_per_side;
328  	int platform_data[2];
329  	struct vchiq_shared_state master;
330  	struct vchiq_shared_state slave;
331  	struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS];
332  };
333  
334  struct vchiq_state {
335  	struct device *dev;
336  	int id;
337  	int initialised;
338  	enum vchiq_connstate conn_state;
339  	short version_common;
340  
341  	struct vchiq_shared_state *local;
342  	struct vchiq_shared_state *remote;
343  	struct vchiq_slot *slot_data;
344  
345  	unsigned short default_slot_quota;
346  	unsigned short default_message_quota;
347  
348  	/* Event indicating connect message received */
349  	struct completion connect;
350  
351  	/* Mutex protecting services */
352  	struct mutex mutex;
353  	struct vchiq_instance **instance;
354  
355  	/* Processes incoming messages */
356  	struct task_struct *slot_handler_thread;
357  
358  	/* Processes recycled slots */
359  	struct task_struct *recycle_thread;
360  
361  	/* Processes synchronous messages */
362  	struct task_struct *sync_thread;
363  
364  	/* Local implementation of the trigger remote event */
365  	wait_queue_head_t trigger_event;
366  
367  	/* Local implementation of the recycle remote event */
368  	wait_queue_head_t recycle_event;
369  
370  	/* Local implementation of the sync trigger remote event */
371  	wait_queue_head_t sync_trigger_event;
372  
373  	/* Local implementation of the sync release remote event */
374  	wait_queue_head_t sync_release_event;
375  
376  	char *tx_data;
377  	char *rx_data;
378  	struct vchiq_slot_info *rx_info;
379  
380  	struct mutex slot_mutex;
381  
382  	struct mutex recycle_mutex;
383  
384  	struct mutex sync_mutex;
385  
386  	struct mutex bulk_transfer_mutex;
387  
388  	/*
389  	 * Indicates the byte position within the stream from where the next
390  	 * message will be read. The least significant bits are an index into
391  	 * the slot.The next bits are the index of the slot in
392  	 * remote->slot_queue.
393  	 */
394  	int rx_pos;
395  
396  	/*
397  	 * A cached copy of local->tx_pos. Only write to local->tx_pos, and read
398  	 * from remote->tx_pos.
399  	 */
400  	int local_tx_pos;
401  
402  	/* The slot_queue index of the slot to become available next. */
403  	int slot_queue_available;
404  
405  	/* A flag to indicate if any poll has been requested */
406  	int poll_needed;
407  
408  	/* Ths index of the previous slot used for data messages. */
409  	int previous_data_index;
410  
411  	/* The number of slots occupied by data messages. */
412  	unsigned short data_use_count;
413  
414  	/* The maximum number of slots to be occupied by data messages. */
415  	unsigned short data_quota;
416  
417  	/* An array of bit sets indicating which services must be polled. */
418  	atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
419  
420  	/* The number of the first unused service */
421  	int unused_service;
422  
423  	/* Signalled when a free slot becomes available. */
424  	struct completion slot_available_event;
425  
426  	struct completion slot_remove_event;
427  
428  	/* Signalled when a free data slot becomes available. */
429  	struct completion data_quota_event;
430  
431  	struct state_stats_struct {
432  		int slot_stalls;
433  		int data_stalls;
434  		int ctrl_tx_count;
435  		int ctrl_rx_count;
436  		int error_count;
437  	} stats;
438  
439  	struct vchiq_service __rcu *services[VCHIQ_MAX_SERVICES];
440  	struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES];
441  	struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS];
442  
443  	struct opaque_platform_state *platform_state;
444  };
445  
446  struct bulk_waiter {
447  	struct vchiq_bulk *bulk;
448  	struct completion event;
449  	int actual;
450  };
451  
452  struct vchiq_config {
453  	unsigned int max_msg_size;
454  	unsigned int bulk_threshold;	/* The message size above which it
455  					 * is better to use a bulk transfer
456  					 * (<= max_msg_size)
457  					 */
458  	unsigned int max_outstanding_bulks;
459  	unsigned int max_services;
460  	short version;      /* The version of VCHIQ */
461  	short version_min;  /* The minimum compatible version of VCHIQ */
462  };
463  
464  extern spinlock_t bulk_waiter_spinlock;
465  
466  extern int vchiq_core_log_level;
467  extern int vchiq_core_msg_log_level;
468  extern int vchiq_sync_log_level;
469  
470  extern const char *
471  get_conn_state_name(enum vchiq_connstate conn_state);
472  
473  extern struct vchiq_slot_zero *
474  vchiq_init_slots(void *mem_base, int mem_size);
475  
476  extern int
477  vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero, struct device *dev);
478  
479  extern int
480  vchiq_connect_internal(struct vchiq_state *state, struct vchiq_instance *instance);
481  
482  struct vchiq_service *
483  vchiq_add_service_internal(struct vchiq_state *state,
484  			   const struct vchiq_service_params_kernel *params,
485  			   int srvstate, struct vchiq_instance *instance,
486  			   void (*userdata_term)(void *userdata));
487  
488  extern int
489  vchiq_open_service_internal(struct vchiq_service *service, int client_id);
490  
491  extern int
492  vchiq_close_service_internal(struct vchiq_service *service, int close_recvd);
493  
494  extern void
495  vchiq_terminate_service_internal(struct vchiq_service *service);
496  
497  extern void
498  vchiq_free_service_internal(struct vchiq_service *service);
499  
500  extern void
501  vchiq_shutdown_internal(struct vchiq_state *state, struct vchiq_instance *instance);
502  
503  extern void
504  remote_event_pollall(struct vchiq_state *state);
505  
506  extern int
507  vchiq_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *offset,
508  		    void __user *uoffset, int size, void *userdata, enum vchiq_bulk_mode mode,
509  		    enum vchiq_bulk_dir dir);
510  
511  extern int
512  vchiq_dump_state(void *dump_context, struct vchiq_state *state);
513  
514  extern int
515  vchiq_dump_service_state(void *dump_context, struct vchiq_service *service);
516  
517  extern void
518  vchiq_loud_error_header(void);
519  
520  extern void
521  vchiq_loud_error_footer(void);
522  
523  extern void
524  request_poll(struct vchiq_state *state, struct vchiq_service *service,
525  	     int poll_type);
526  
527  struct vchiq_service *handle_to_service(struct vchiq_instance *instance, unsigned int handle);
528  
529  extern struct vchiq_service *
530  find_service_by_handle(struct vchiq_instance *instance, unsigned int handle);
531  
532  extern struct vchiq_service *
533  find_service_by_port(struct vchiq_state *state, unsigned int localport);
534  
535  extern struct vchiq_service *
536  find_service_for_instance(struct vchiq_instance *instance, unsigned int handle);
537  
538  extern struct vchiq_service *
539  find_closed_service_for_instance(struct vchiq_instance *instance, unsigned int handle);
540  
541  extern struct vchiq_service *
542  __next_service_by_instance(struct vchiq_state *state,
543  			   struct vchiq_instance *instance,
544  			   int *pidx);
545  
546  extern struct vchiq_service *
547  next_service_by_instance(struct vchiq_state *state,
548  			 struct vchiq_instance *instance,
549  			 int *pidx);
550  
551  extern void
552  vchiq_service_get(struct vchiq_service *service);
553  
554  extern void
555  vchiq_service_put(struct vchiq_service *service);
556  
557  extern int
558  vchiq_queue_message(struct vchiq_instance *instance, unsigned int handle,
559  		    ssize_t (*copy_callback)(void *context, void *dest,
560  					     size_t offset, size_t maxsize),
561  		    void *context,
562  		    size_t size);
563  
564  int vchiq_prepare_bulk_data(struct vchiq_instance *instance, struct vchiq_bulk *bulk, void *offset,
565  			    void __user *uoffset, int size, int dir);
566  
567  void vchiq_complete_bulk(struct vchiq_instance *instance, struct vchiq_bulk *bulk);
568  
569  void remote_event_signal(struct remote_event *event);
570  
571  int vchiq_dump(void *dump_context, const char *str, int len);
572  
573  int vchiq_dump_platform_state(void *dump_context);
574  
575  int vchiq_dump_platform_instances(void *dump_context);
576  
577  int vchiq_dump_platform_service_state(void *dump_context, struct vchiq_service *service);
578  
579  int vchiq_use_service_internal(struct vchiq_service *service);
580  
581  int vchiq_release_service_internal(struct vchiq_service *service);
582  
583  void vchiq_on_remote_use(struct vchiq_state *state);
584  
585  void vchiq_on_remote_release(struct vchiq_state *state);
586  
587  int vchiq_platform_init_state(struct vchiq_state *state);
588  
589  int vchiq_check_service(struct vchiq_service *service);
590  
591  void vchiq_on_remote_use_active(struct vchiq_state *state);
592  
593  int vchiq_send_remote_use(struct vchiq_state *state);
594  
595  int vchiq_send_remote_use_active(struct vchiq_state *state);
596  
597  void vchiq_platform_conn_state_changed(struct vchiq_state *state,
598  				       enum vchiq_connstate oldstate,
599  				  enum vchiq_connstate newstate);
600  
601  void vchiq_set_conn_state(struct vchiq_state *state, enum vchiq_connstate newstate);
602  
603  void vchiq_log_dump_mem(const char *label, u32 addr, const void *void_mem, size_t num_bytes);
604  
605  int vchiq_remove_service(struct vchiq_instance *instance, unsigned int service);
606  
607  int vchiq_get_client_id(struct vchiq_instance *instance, unsigned int service);
608  
609  void vchiq_get_config(struct vchiq_config *config);
610  
611  int vchiq_set_service_option(struct vchiq_instance *instance, unsigned int service,
612  			     enum vchiq_service_option option, int value);
613  
614  #endif
615