1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /*
3  * Copyright (C) 2005-2014, 2018-2021 Intel Corporation
4  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
5  * Copyright (C) 2016-2017 Intel Deutschland GmbH
6  */
7 #ifndef __iwl_trans_h__
8 #define __iwl_trans_h__
9 
10 #include <linux/ieee80211.h>
11 #include <linux/mm.h> /* for page_address */
12 #include <linux/lockdep.h>
13 #include <linux/kernel.h>
14 
15 #include "iwl-debug.h"
16 #include "iwl-config.h"
17 #include "fw/img.h"
18 #include "iwl-op-mode.h"
19 #include <linux/firmware.h>
20 #include "fw/api/cmdhdr.h"
21 #include "fw/api/txq.h"
22 #include "fw/api/dbg-tlv.h"
23 #include "iwl-dbg-tlv.h"
24 
25 /**
26  * DOC: Transport layer - what is it ?
27  *
28  * The transport layer is the layer that deals with the HW directly. It provides
29  * an abstraction of the underlying HW to the upper layer. The transport layer
30  * doesn't provide any policy, algorithm or anything of this kind, but only
31  * mechanisms to make the HW do something. It is not completely stateless but
32  * close to it.
33  * We will have an implementation for each different supported bus.
34  */
35 
36 /**
37  * DOC: Life cycle of the transport layer
38  *
39  * The transport layer has a very precise life cycle.
40  *
41  *	1) A helper function is called during the module initialization and
42  *	   registers the bus driver's ops with the transport's alloc function.
43  *	2) Bus's probe calls to the transport layer's allocation functions.
44  *	   Of course this function is bus specific.
45  *	3) This allocation functions will spawn the upper layer which will
46  *	   register mac80211.
47  *
48  *	4) At some point (i.e. mac80211's start call), the op_mode will call
49  *	   the following sequence:
50  *	   start_hw
51  *	   start_fw
52  *
53  *	5) Then when finished (or reset):
54  *	   stop_device
55  *
56  *	6) Eventually, the free function will be called.
57  */
58 
59 #define IWL_TRANS_FW_DBG_DOMAIN(trans)	IWL_FW_INI_DOMAIN_ALWAYS_ON
60 
61 #define FH_RSCSR_FRAME_SIZE_MSK		0x00003FFF	/* bits 0-13 */
62 #define FH_RSCSR_FRAME_INVALID		0x55550000
63 #define FH_RSCSR_FRAME_ALIGN		0x40
64 #define FH_RSCSR_RPA_EN			BIT(25)
65 #define FH_RSCSR_RADA_EN		BIT(26)
66 #define FH_RSCSR_RXQ_POS		16
67 #define FH_RSCSR_RXQ_MASK		0x3F0000
68 
69 struct iwl_rx_packet {
70 	/*
71 	 * The first 4 bytes of the RX frame header contain both the RX frame
72 	 * size and some flags.
73 	 * Bit fields:
74 	 * 31:    flag flush RB request
75 	 * 30:    flag ignore TC (terminal counter) request
76 	 * 29:    flag fast IRQ request
77 	 * 28-27: Reserved
78 	 * 26:    RADA enabled
79 	 * 25:    Offload enabled
80 	 * 24:    RPF enabled
81 	 * 23:    RSS enabled
82 	 * 22:    Checksum enabled
83 	 * 21-16: RX queue
84 	 * 15-14: Reserved
85 	 * 13-00: RX frame size
86 	 */
87 	__le32 len_n_flags;
88 	struct iwl_cmd_header hdr;
89 	u8 data[];
90 } __packed;
91 
92 static inline u32 iwl_rx_packet_len(const struct iwl_rx_packet *pkt)
93 {
94 	return le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_FRAME_SIZE_MSK;
95 }
96 
97 static inline u32 iwl_rx_packet_payload_len(const struct iwl_rx_packet *pkt)
98 {
99 	return iwl_rx_packet_len(pkt) - sizeof(pkt->hdr);
100 }
101 
102 /**
103  * enum CMD_MODE - how to send the host commands ?
104  *
105  * @CMD_ASYNC: Return right away and don't wait for the response
106  * @CMD_WANT_SKB: Not valid with CMD_ASYNC. The caller needs the buffer of
107  *	the response. The caller needs to call iwl_free_resp when done.
108  * @CMD_WANT_ASYNC_CALLBACK: the op_mode's async callback function must be
109  *	called after this command completes. Valid only with CMD_ASYNC.
110  * @CMD_SEND_IN_D3: Allow the command to be sent in D3 mode, relevant to
111  *	SUSPEND and RESUME commands. We are in D3 mode when we set
112  *	trans->system_pm_mode to IWL_PLAT_PM_MODE_D3.
113  */
114 enum CMD_MODE {
115 	CMD_ASYNC		= BIT(0),
116 	CMD_WANT_SKB		= BIT(1),
117 	CMD_SEND_IN_RFKILL	= BIT(2),
118 	CMD_WANT_ASYNC_CALLBACK	= BIT(3),
119 	CMD_SEND_IN_D3          = BIT(4),
120 };
121 
122 #define DEF_CMD_PAYLOAD_SIZE 320
123 
124 /**
125  * struct iwl_device_cmd
126  *
127  * For allocation of the command and tx queues, this establishes the overall
128  * size of the largest command we send to uCode, except for commands that
129  * aren't fully copied and use other TFD space.
130  */
131 struct iwl_device_cmd {
132 	union {
133 		struct {
134 			struct iwl_cmd_header hdr;	/* uCode API */
135 			u8 payload[DEF_CMD_PAYLOAD_SIZE];
136 		};
137 		struct {
138 			struct iwl_cmd_header_wide hdr_wide;
139 			u8 payload_wide[DEF_CMD_PAYLOAD_SIZE -
140 					sizeof(struct iwl_cmd_header_wide) +
141 					sizeof(struct iwl_cmd_header)];
142 		};
143 	};
144 } __packed;
145 
146 /**
147  * struct iwl_device_tx_cmd - buffer for TX command
148  * @hdr: the header
149  * @payload: the payload placeholder
150  *
151  * The actual structure is sized dynamically according to need.
152  */
153 struct iwl_device_tx_cmd {
154 	struct iwl_cmd_header hdr;
155 	u8 payload[];
156 } __packed;
157 
158 #define TFD_MAX_PAYLOAD_SIZE (sizeof(struct iwl_device_cmd))
159 
160 /*
161  * number of transfer buffers (fragments) per transmit frame descriptor;
162  * this is just the driver's idea, the hardware supports 20
163  */
164 #define IWL_MAX_CMD_TBS_PER_TFD	2
165 
166 /* We need 2 entries for the TX command and header, and another one might
167  * be needed for potential data in the SKB's head. The remaining ones can
168  * be used for frags.
169  */
170 #define IWL_TRANS_MAX_FRAGS(trans) ((trans)->txqs.tfd.max_tbs - 3)
171 
172 /**
173  * enum iwl_hcmd_dataflag - flag for each one of the chunks of the command
174  *
175  * @IWL_HCMD_DFL_NOCOPY: By default, the command is copied to the host command's
176  *	ring. The transport layer doesn't map the command's buffer to DMA, but
177  *	rather copies it to a previously allocated DMA buffer. This flag tells
178  *	the transport layer not to copy the command, but to map the existing
179  *	buffer (that is passed in) instead. This saves the memcpy and allows
180  *	commands that are bigger than the fixed buffer to be submitted.
181  *	Note that a TFD entry after a NOCOPY one cannot be a normal copied one.
182  * @IWL_HCMD_DFL_DUP: Only valid without NOCOPY, duplicate the memory for this
183  *	chunk internally and free it again after the command completes. This
184  *	can (currently) be used only once per command.
185  *	Note that a TFD entry after a DUP one cannot be a normal copied one.
186  */
187 enum iwl_hcmd_dataflag {
188 	IWL_HCMD_DFL_NOCOPY	= BIT(0),
189 	IWL_HCMD_DFL_DUP	= BIT(1),
190 };
191 
192 enum iwl_error_event_table_status {
193 	IWL_ERROR_EVENT_TABLE_LMAC1 = BIT(0),
194 	IWL_ERROR_EVENT_TABLE_LMAC2 = BIT(1),
195 	IWL_ERROR_EVENT_TABLE_UMAC = BIT(2),
196 	IWL_ERROR_EVENT_TABLE_TCM = BIT(3),
197 };
198 
199 /**
200  * struct iwl_host_cmd - Host command to the uCode
201  *
202  * @data: array of chunks that composes the data of the host command
203  * @resp_pkt: response packet, if %CMD_WANT_SKB was set
204  * @_rx_page_order: (internally used to free response packet)
205  * @_rx_page_addr: (internally used to free response packet)
206  * @flags: can be CMD_*
207  * @len: array of the lengths of the chunks in data
208  * @dataflags: IWL_HCMD_DFL_*
209  * @id: command id of the host command, for wide commands encoding the
210  *	version and group as well
211  */
212 struct iwl_host_cmd {
213 	const void *data[IWL_MAX_CMD_TBS_PER_TFD];
214 	struct iwl_rx_packet *resp_pkt;
215 	unsigned long _rx_page_addr;
216 	u32 _rx_page_order;
217 
218 	u32 flags;
219 	u32 id;
220 	u16 len[IWL_MAX_CMD_TBS_PER_TFD];
221 	u8 dataflags[IWL_MAX_CMD_TBS_PER_TFD];
222 };
223 
224 static inline void iwl_free_resp(struct iwl_host_cmd *cmd)
225 {
226 	free_pages(cmd->_rx_page_addr, cmd->_rx_page_order);
227 }
228 
229 struct iwl_rx_cmd_buffer {
230 	struct page *_page;
231 	int _offset;
232 	bool _page_stolen;
233 	u32 _rx_page_order;
234 	unsigned int truesize;
235 };
236 
237 static inline void *rxb_addr(struct iwl_rx_cmd_buffer *r)
238 {
239 	return (void *)((unsigned long)page_address(r->_page) + r->_offset);
240 }
241 
242 static inline int rxb_offset(struct iwl_rx_cmd_buffer *r)
243 {
244 	return r->_offset;
245 }
246 
247 static inline struct page *rxb_steal_page(struct iwl_rx_cmd_buffer *r)
248 {
249 	r->_page_stolen = true;
250 	get_page(r->_page);
251 	return r->_page;
252 }
253 
254 static inline void iwl_free_rxb(struct iwl_rx_cmd_buffer *r)
255 {
256 	__free_pages(r->_page, r->_rx_page_order);
257 }
258 
259 #define MAX_NO_RECLAIM_CMDS	6
260 
261 #define IWL_MASK(lo, hi) ((1 << (hi)) | ((1 << (hi)) - (1 << (lo))))
262 
263 /*
264  * Maximum number of HW queues the transport layer
265  * currently supports
266  */
267 #define IWL_MAX_HW_QUEUES		32
268 #define IWL_MAX_TVQM_QUEUES		512
269 
270 #define IWL_MAX_TID_COUNT	8
271 #define IWL_MGMT_TID		15
272 #define IWL_FRAME_LIMIT	64
273 #define IWL_MAX_RX_HW_QUEUES	16
274 #define IWL_9000_MAX_RX_HW_QUEUES	6
275 
276 /**
277  * enum iwl_wowlan_status - WoWLAN image/device status
278  * @IWL_D3_STATUS_ALIVE: firmware is still running after resume
279  * @IWL_D3_STATUS_RESET: device was reset while suspended
280  */
281 enum iwl_d3_status {
282 	IWL_D3_STATUS_ALIVE,
283 	IWL_D3_STATUS_RESET,
284 };
285 
286 /**
287  * enum iwl_trans_status: transport status flags
288  * @STATUS_SYNC_HCMD_ACTIVE: a SYNC command is being processed
289  * @STATUS_DEVICE_ENABLED: APM is enabled
290  * @STATUS_TPOWER_PMI: the device might be asleep (need to wake it up)
291  * @STATUS_INT_ENABLED: interrupts are enabled
292  * @STATUS_RFKILL_HW: the actual HW state of the RF-kill switch
293  * @STATUS_RFKILL_OPMODE: RF-kill state reported to opmode
294  * @STATUS_FW_ERROR: the fw is in error state
295  * @STATUS_TRANS_GOING_IDLE: shutting down the trans, only special commands
296  *	are sent
297  * @STATUS_TRANS_IDLE: the trans is idle - general commands are not to be sent
298  * @STATUS_TRANS_DEAD: trans is dead - avoid any read/write operation
299  */
300 enum iwl_trans_status {
301 	STATUS_SYNC_HCMD_ACTIVE,
302 	STATUS_DEVICE_ENABLED,
303 	STATUS_TPOWER_PMI,
304 	STATUS_INT_ENABLED,
305 	STATUS_RFKILL_HW,
306 	STATUS_RFKILL_OPMODE,
307 	STATUS_FW_ERROR,
308 	STATUS_TRANS_GOING_IDLE,
309 	STATUS_TRANS_IDLE,
310 	STATUS_TRANS_DEAD,
311 };
312 
313 static inline int
314 iwl_trans_get_rb_size_order(enum iwl_amsdu_size rb_size)
315 {
316 	switch (rb_size) {
317 	case IWL_AMSDU_2K:
318 		return get_order(2 * 1024);
319 	case IWL_AMSDU_4K:
320 		return get_order(4 * 1024);
321 	case IWL_AMSDU_8K:
322 		return get_order(8 * 1024);
323 	case IWL_AMSDU_12K:
324 		return get_order(16 * 1024);
325 	default:
326 		WARN_ON(1);
327 		return -1;
328 	}
329 }
330 
331 static inline int
332 iwl_trans_get_rb_size(enum iwl_amsdu_size rb_size)
333 {
334 	switch (rb_size) {
335 	case IWL_AMSDU_2K:
336 		return 2 * 1024;
337 	case IWL_AMSDU_4K:
338 		return 4 * 1024;
339 	case IWL_AMSDU_8K:
340 		return 8 * 1024;
341 	case IWL_AMSDU_12K:
342 		return 16 * 1024;
343 	default:
344 		WARN_ON(1);
345 		return 0;
346 	}
347 }
348 
349 struct iwl_hcmd_names {
350 	u8 cmd_id;
351 	const char *const cmd_name;
352 };
353 
354 #define HCMD_NAME(x)	\
355 	{ .cmd_id = x, .cmd_name = #x }
356 
357 struct iwl_hcmd_arr {
358 	const struct iwl_hcmd_names *arr;
359 	int size;
360 };
361 
362 #define HCMD_ARR(x)	\
363 	{ .arr = x, .size = ARRAY_SIZE(x) }
364 
365 /**
366  * struct iwl_trans_config - transport configuration
367  *
368  * @op_mode: pointer to the upper layer.
369  * @cmd_queue: the index of the command queue.
370  *	Must be set before start_fw.
371  * @cmd_fifo: the fifo for host commands
372  * @cmd_q_wdg_timeout: the timeout of the watchdog timer for the command queue.
373  * @no_reclaim_cmds: Some devices erroneously don't set the
374  *	SEQ_RX_FRAME bit on some notifications, this is the
375  *	list of such notifications to filter. Max length is
376  *	%MAX_NO_RECLAIM_CMDS.
377  * @n_no_reclaim_cmds: # of commands in list
378  * @rx_buf_size: RX buffer size needed for A-MSDUs
379  *	if unset 4k will be the RX buffer size
380  * @bc_table_dword: set to true if the BC table expects the byte count to be
381  *	in DWORD (as opposed to bytes)
382  * @scd_set_active: should the transport configure the SCD for HCMD queue
383  * @command_groups: array of command groups, each member is an array of the
384  *	commands in the group; for debugging only
385  * @command_groups_size: number of command groups, to avoid illegal access
386  * @cb_data_offs: offset inside skb->cb to store transport data at, must have
387  *	space for at least two pointers
388  * @fw_reset_handshake: firmware supports reset flow handshake
389  */
390 struct iwl_trans_config {
391 	struct iwl_op_mode *op_mode;
392 
393 	u8 cmd_queue;
394 	u8 cmd_fifo;
395 	unsigned int cmd_q_wdg_timeout;
396 	const u8 *no_reclaim_cmds;
397 	unsigned int n_no_reclaim_cmds;
398 
399 	enum iwl_amsdu_size rx_buf_size;
400 	bool bc_table_dword;
401 	bool scd_set_active;
402 	const struct iwl_hcmd_arr *command_groups;
403 	int command_groups_size;
404 
405 	u8 cb_data_offs;
406 	bool fw_reset_handshake;
407 };
408 
409 struct iwl_trans_dump_data {
410 	u32 len;
411 	u8 data[];
412 };
413 
414 struct iwl_trans;
415 
416 struct iwl_trans_txq_scd_cfg {
417 	u8 fifo;
418 	u8 sta_id;
419 	u8 tid;
420 	bool aggregate;
421 	int frame_limit;
422 };
423 
424 /**
425  * struct iwl_trans_rxq_dma_data - RX queue DMA data
426  * @fr_bd_cb: DMA address of free BD cyclic buffer
427  * @fr_bd_wid: Initial write index of the free BD cyclic buffer
428  * @urbd_stts_wrptr: DMA address of urbd_stts_wrptr
429  * @ur_bd_cb: DMA address of used BD cyclic buffer
430  */
431 struct iwl_trans_rxq_dma_data {
432 	u64 fr_bd_cb;
433 	u32 fr_bd_wid;
434 	u64 urbd_stts_wrptr;
435 	u64 ur_bd_cb;
436 };
437 
438 /**
439  * struct iwl_trans_ops - transport specific operations
440  *
441  * All the handlers MUST be implemented
442  *
443  * @start_hw: starts the HW. From that point on, the HW can send interrupts.
444  *	May sleep.
445  * @op_mode_leave: Turn off the HW RF kill indication if on
446  *	May sleep
447  * @start_fw: allocates and inits all the resources for the transport
448  *	layer. Also kick a fw image.
449  *	May sleep
450  * @fw_alive: called when the fw sends alive notification. If the fw provides
451  *	the SCD base address in SRAM, then provide it here, or 0 otherwise.
452  *	May sleep
453  * @stop_device: stops the whole device (embedded CPU put to reset) and stops
454  *	the HW. From that point on, the HW will be stopped but will still issue
455  *	an interrupt if the HW RF kill switch is triggered.
456  *	This callback must do the right thing and not crash even if %start_hw()
457  *	was called but not &start_fw(). May sleep.
458  * @d3_suspend: put the device into the correct mode for WoWLAN during
459  *	suspend. This is optional, if not implemented WoWLAN will not be
460  *	supported. This callback may sleep.
461  * @d3_resume: resume the device after WoWLAN, enabling the opmode to
462  *	talk to the WoWLAN image to get its status. This is optional, if not
463  *	implemented WoWLAN will not be supported. This callback may sleep.
464  * @send_cmd:send a host command. Must return -ERFKILL if RFkill is asserted.
465  *	If RFkill is asserted in the middle of a SYNC host command, it must
466  *	return -ERFKILL straight away.
467  *	May sleep only if CMD_ASYNC is not set
468  * @tx: send an skb. The transport relies on the op_mode to zero the
469  *	the ieee80211_tx_info->driver_data. If the MPDU is an A-MSDU, all
470  *	the CSUM will be taken care of (TCP CSUM and IP header in case of
471  *	IPv4). If the MPDU is a single MSDU, the op_mode must compute the IP
472  *	header if it is IPv4.
473  *	Must be atomic
474  * @reclaim: free packet until ssn. Returns a list of freed packets.
475  *	Must be atomic
476  * @txq_enable: setup a queue. To setup an AC queue, use the
477  *	iwl_trans_ac_txq_enable wrapper. fw_alive must have been called before
478  *	this one. The op_mode must not configure the HCMD queue. The scheduler
479  *	configuration may be %NULL, in which case the hardware will not be
480  *	configured. If true is returned, the operation mode needs to increment
481  *	the sequence number of the packets routed to this queue because of a
482  *	hardware scheduler bug. May sleep.
483  * @txq_disable: de-configure a Tx queue to send AMPDUs
484  *	Must be atomic
485  * @txq_set_shared_mode: change Tx queue shared/unshared marking
486  * @wait_tx_queues_empty: wait until tx queues are empty. May sleep.
487  * @wait_txq_empty: wait until specific tx queue is empty. May sleep.
488  * @freeze_txq_timer: prevents the timer of the queue from firing until the
489  *	queue is set to awake. Must be atomic.
490  * @block_txq_ptrs: stop updating the write pointers of the Tx queues. Note
491  *	that the transport needs to refcount the calls since this function
492  *	will be called several times with block = true, and then the queues
493  *	need to be unblocked only after the same number of calls with
494  *	block = false.
495  * @write8: write a u8 to a register at offset ofs from the BAR
496  * @write32: write a u32 to a register at offset ofs from the BAR
497  * @read32: read a u32 register at offset ofs from the BAR
498  * @read_prph: read a DWORD from a periphery register
499  * @write_prph: write a DWORD to a periphery register
500  * @read_mem: read device's SRAM in DWORD
501  * @write_mem: write device's SRAM in DWORD. If %buf is %NULL, then the memory
502  *	will be zeroed.
503  * @read_config32: read a u32 value from the device's config space at
504  *	the given offset.
505  * @configure: configure parameters required by the transport layer from
506  *	the op_mode. May be called several times before start_fw, can't be
507  *	called after that.
508  * @set_pmi: set the power pmi state
509  * @grab_nic_access: wake the NIC to be able to access non-HBUS regs.
510  *	Sleeping is not allowed between grab_nic_access and
511  *	release_nic_access.
512  * @release_nic_access: let the NIC go to sleep. The "flags" parameter
513  *	must be the same one that was sent before to the grab_nic_access.
514  * @set_bits_mask - set SRAM register according to value and mask.
515  * @dump_data: return a vmalloc'ed buffer with debug data, maybe containing last
516  *	TX'ed commands and similar. The buffer will be vfree'd by the caller.
517  *	Note that the transport must fill in the proper file headers.
518  * @debugfs_cleanup: used in the driver unload flow to make a proper cleanup
519  *	of the trans debugfs
520  * @set_pnvm: set the pnvm data in the prph scratch buffer, inside the
521  *	context info.
522  * @interrupts: disable/enable interrupts to transport
523  */
524 struct iwl_trans_ops {
525 
526 	int (*start_hw)(struct iwl_trans *iwl_trans);
527 	void (*op_mode_leave)(struct iwl_trans *iwl_trans);
528 	int (*start_fw)(struct iwl_trans *trans, const struct fw_img *fw,
529 			bool run_in_rfkill);
530 	void (*fw_alive)(struct iwl_trans *trans, u32 scd_addr);
531 	void (*stop_device)(struct iwl_trans *trans);
532 
533 	int (*d3_suspend)(struct iwl_trans *trans, bool test, bool reset);
534 	int (*d3_resume)(struct iwl_trans *trans, enum iwl_d3_status *status,
535 			 bool test, bool reset);
536 
537 	int (*send_cmd)(struct iwl_trans *trans, struct iwl_host_cmd *cmd);
538 
539 	int (*tx)(struct iwl_trans *trans, struct sk_buff *skb,
540 		  struct iwl_device_tx_cmd *dev_cmd, int queue);
541 	void (*reclaim)(struct iwl_trans *trans, int queue, int ssn,
542 			struct sk_buff_head *skbs);
543 
544 	void (*set_q_ptrs)(struct iwl_trans *trans, int queue, int ptr);
545 
546 	bool (*txq_enable)(struct iwl_trans *trans, int queue, u16 ssn,
547 			   const struct iwl_trans_txq_scd_cfg *cfg,
548 			   unsigned int queue_wdg_timeout);
549 	void (*txq_disable)(struct iwl_trans *trans, int queue,
550 			    bool configure_scd);
551 	/* 22000 functions */
552 	int (*txq_alloc)(struct iwl_trans *trans,
553 			 __le16 flags, u8 sta_id, u8 tid,
554 			 int cmd_id, int size,
555 			 unsigned int queue_wdg_timeout);
556 	void (*txq_free)(struct iwl_trans *trans, int queue);
557 	int (*rxq_dma_data)(struct iwl_trans *trans, int queue,
558 			    struct iwl_trans_rxq_dma_data *data);
559 
560 	void (*txq_set_shared_mode)(struct iwl_trans *trans, u32 txq_id,
561 				    bool shared);
562 
563 	int (*wait_tx_queues_empty)(struct iwl_trans *trans, u32 txq_bm);
564 	int (*wait_txq_empty)(struct iwl_trans *trans, int queue);
565 	void (*freeze_txq_timer)(struct iwl_trans *trans, unsigned long txqs,
566 				 bool freeze);
567 	void (*block_txq_ptrs)(struct iwl_trans *trans, bool block);
568 
569 	void (*write8)(struct iwl_trans *trans, u32 ofs, u8 val);
570 	void (*write32)(struct iwl_trans *trans, u32 ofs, u32 val);
571 	u32 (*read32)(struct iwl_trans *trans, u32 ofs);
572 	u32 (*read_prph)(struct iwl_trans *trans, u32 ofs);
573 	void (*write_prph)(struct iwl_trans *trans, u32 ofs, u32 val);
574 	int (*read_mem)(struct iwl_trans *trans, u32 addr,
575 			void *buf, int dwords);
576 	int (*write_mem)(struct iwl_trans *trans, u32 addr,
577 			 const void *buf, int dwords);
578 	int (*read_config32)(struct iwl_trans *trans, u32 ofs, u32 *val);
579 	void (*configure)(struct iwl_trans *trans,
580 			  const struct iwl_trans_config *trans_cfg);
581 	void (*set_pmi)(struct iwl_trans *trans, bool state);
582 	void (*sw_reset)(struct iwl_trans *trans);
583 	bool (*grab_nic_access)(struct iwl_trans *trans);
584 	void (*release_nic_access)(struct iwl_trans *trans);
585 	void (*set_bits_mask)(struct iwl_trans *trans, u32 reg, u32 mask,
586 			      u32 value);
587 
588 	struct iwl_trans_dump_data *(*dump_data)(struct iwl_trans *trans,
589 						 u32 dump_mask);
590 	void (*debugfs_cleanup)(struct iwl_trans *trans);
591 	void (*sync_nmi)(struct iwl_trans *trans);
592 	int (*set_pnvm)(struct iwl_trans *trans, const void *data, u32 len);
593 	int (*set_reduce_power)(struct iwl_trans *trans,
594 				const void *data, u32 len);
595 	void (*interrupts)(struct iwl_trans *trans, bool enable);
596 };
597 
598 /**
599  * enum iwl_trans_state - state of the transport layer
600  *
601  * @IWL_TRANS_NO_FW: firmware wasn't started yet, or crashed
602  * @IWL_TRANS_FW_STARTED: FW was started, but not alive yet
603  * @IWL_TRANS_FW_ALIVE: FW has sent an alive response
604  */
605 enum iwl_trans_state {
606 	IWL_TRANS_NO_FW,
607 	IWL_TRANS_FW_STARTED,
608 	IWL_TRANS_FW_ALIVE,
609 };
610 
611 /**
612  * DOC: Platform power management
613  *
614  * In system-wide power management the entire platform goes into a low
615  * power state (e.g. idle or suspend to RAM) at the same time and the
616  * device is configured as a wakeup source for the entire platform.
617  * This is usually triggered by userspace activity (e.g. the user
618  * presses the suspend button or a power management daemon decides to
619  * put the platform in low power mode).  The device's behavior in this
620  * mode is dictated by the wake-on-WLAN configuration.
621  *
622  * The terms used for the device's behavior are as follows:
623  *
624  *	- D0: the device is fully powered and the host is awake;
625  *	- D3: the device is in low power mode and only reacts to
626  *		specific events (e.g. magic-packet received or scan
627  *		results found);
628  *
629  * These terms reflect the power modes in the firmware and are not to
630  * be confused with the physical device power state.
631  */
632 
633 /**
634  * enum iwl_plat_pm_mode - platform power management mode
635  *
636  * This enumeration describes the device's platform power management
637  * behavior when in system-wide suspend (i.e WoWLAN).
638  *
639  * @IWL_PLAT_PM_MODE_DISABLED: power management is disabled for this
640  *	device.  In system-wide suspend mode, it means that the all
641  *	connections will be closed automatically by mac80211 before
642  *	the platform is suspended.
643  * @IWL_PLAT_PM_MODE_D3: the device goes into D3 mode (i.e. WoWLAN).
644  */
645 enum iwl_plat_pm_mode {
646 	IWL_PLAT_PM_MODE_DISABLED,
647 	IWL_PLAT_PM_MODE_D3,
648 };
649 
650 /**
651  * enum iwl_ini_cfg_state
652  * @IWL_INI_CFG_STATE_NOT_LOADED: no debug cfg was given
653  * @IWL_INI_CFG_STATE_LOADED: debug cfg was found and loaded
654  * @IWL_INI_CFG_STATE_CORRUPTED: debug cfg was found and some of the TLVs
655  *	are corrupted. The rest of the debug TLVs will still be used
656  */
657 enum iwl_ini_cfg_state {
658 	IWL_INI_CFG_STATE_NOT_LOADED,
659 	IWL_INI_CFG_STATE_LOADED,
660 	IWL_INI_CFG_STATE_CORRUPTED,
661 };
662 
663 /* Max time to wait for nmi interrupt */
664 #define IWL_TRANS_NMI_TIMEOUT (HZ / 4)
665 
666 /**
667  * struct iwl_dram_data
668  * @physical: page phy pointer
669  * @block: pointer to the allocated block/page
670  * @size: size of the block/page
671  */
672 struct iwl_dram_data {
673 	dma_addr_t physical;
674 	void *block;
675 	int size;
676 };
677 
678 /**
679  * struct iwl_fw_mon - fw monitor per allocation id
680  * @num_frags: number of fragments
681  * @frags: an array of DRAM buffer fragments
682  */
683 struct iwl_fw_mon {
684 	u32 num_frags;
685 	struct iwl_dram_data *frags;
686 };
687 
688 /**
689  * struct iwl_self_init_dram - dram data used by self init process
690  * @fw: lmac and umac dram data
691  * @fw_cnt: total number of items in array
692  * @paging: paging dram data
693  * @paging_cnt: total number of items in array
694  */
695 struct iwl_self_init_dram {
696 	struct iwl_dram_data *fw;
697 	int fw_cnt;
698 	struct iwl_dram_data *paging;
699 	int paging_cnt;
700 };
701 
702 /**
703  * struct iwl_trans_debug - transport debug related data
704  *
705  * @n_dest_reg: num of reg_ops in %dbg_dest_tlv
706  * @rec_on: true iff there is a fw debug recording currently active
707  * @dest_tlv: points to the destination TLV for debug
708  * @conf_tlv: array of pointers to configuration TLVs for debug
709  * @trigger_tlv: array of pointers to triggers TLVs for debug
710  * @lmac_error_event_table: addrs of lmacs error tables
711  * @umac_error_event_table: addr of umac error table
712  * @tcm_error_event_table: address of TCM error table
713  * @error_event_table_tlv_status: bitmap that indicates what error table
714  *	pointers was recevied via TLV. uses enum &iwl_error_event_table_status
715  * @internal_ini_cfg: internal debug cfg state. Uses &enum iwl_ini_cfg_state
716  * @external_ini_cfg: external debug cfg state. Uses &enum iwl_ini_cfg_state
717  * @fw_mon_cfg: debug buffer allocation configuration
718  * @fw_mon_ini: DRAM buffer fragments per allocation id
719  * @fw_mon: DRAM buffer for firmware monitor
720  * @hw_error: equals true if hw error interrupt was received from the FW
721  * @ini_dest: debug monitor destination uses &enum iwl_fw_ini_buffer_location
722  * @active_regions: active regions
723  * @debug_info_tlv_list: list of debug info TLVs
724  * @time_point: array of debug time points
725  * @periodic_trig_list: periodic triggers list
726  * @domains_bitmap: bitmap of active domains other than
727  *	&IWL_FW_INI_DOMAIN_ALWAYS_ON
728  */
729 struct iwl_trans_debug {
730 	u8 n_dest_reg;
731 	bool rec_on;
732 
733 	const struct iwl_fw_dbg_dest_tlv_v1 *dest_tlv;
734 	const struct iwl_fw_dbg_conf_tlv *conf_tlv[FW_DBG_CONF_MAX];
735 	struct iwl_fw_dbg_trigger_tlv * const *trigger_tlv;
736 
737 	u32 lmac_error_event_table[2];
738 	u32 umac_error_event_table;
739 	u32 tcm_error_event_table;
740 	unsigned int error_event_table_tlv_status;
741 
742 	enum iwl_ini_cfg_state internal_ini_cfg;
743 	enum iwl_ini_cfg_state external_ini_cfg;
744 
745 	struct iwl_fw_ini_allocation_tlv fw_mon_cfg[IWL_FW_INI_ALLOCATION_NUM];
746 	struct iwl_fw_mon fw_mon_ini[IWL_FW_INI_ALLOCATION_NUM];
747 
748 	struct iwl_dram_data fw_mon;
749 
750 	bool hw_error;
751 	enum iwl_fw_ini_buffer_location ini_dest;
752 
753 	u64 unsupported_region_msk;
754 	struct iwl_ucode_tlv *active_regions[IWL_FW_INI_MAX_REGION_ID];
755 	struct list_head debug_info_tlv_list;
756 	struct iwl_dbg_tlv_time_point_data
757 		time_point[IWL_FW_INI_TIME_POINT_NUM];
758 	struct list_head periodic_trig_list;
759 
760 	u32 domains_bitmap;
761 };
762 
763 struct iwl_dma_ptr {
764 	dma_addr_t dma;
765 	void *addr;
766 	size_t size;
767 };
768 
769 struct iwl_cmd_meta {
770 	/* only for SYNC commands, iff the reply skb is wanted */
771 	struct iwl_host_cmd *source;
772 	u32 flags;
773 	u32 tbs;
774 };
775 
776 /*
777  * The FH will write back to the first TB only, so we need to copy some data
778  * into the buffer regardless of whether it should be mapped or not.
779  * This indicates how big the first TB must be to include the scratch buffer
780  * and the assigned PN.
781  * Since PN location is 8 bytes at offset 12, it's 20 now.
782  * If we make it bigger then allocations will be bigger and copy slower, so
783  * that's probably not useful.
784  */
785 #define IWL_FIRST_TB_SIZE	20
786 #define IWL_FIRST_TB_SIZE_ALIGN ALIGN(IWL_FIRST_TB_SIZE, 64)
787 
788 struct iwl_pcie_txq_entry {
789 	void *cmd;
790 	struct sk_buff *skb;
791 	/* buffer to free after command completes */
792 	const void *free_buf;
793 	struct iwl_cmd_meta meta;
794 };
795 
796 struct iwl_pcie_first_tb_buf {
797 	u8 buf[IWL_FIRST_TB_SIZE_ALIGN];
798 };
799 
800 /**
801  * struct iwl_txq - Tx Queue for DMA
802  * @q: generic Rx/Tx queue descriptor
803  * @tfds: transmit frame descriptors (DMA memory)
804  * @first_tb_bufs: start of command headers, including scratch buffers, for
805  *	the writeback -- this is DMA memory and an array holding one buffer
806  *	for each command on the queue
807  * @first_tb_dma: DMA address for the first_tb_bufs start
808  * @entries: transmit entries (driver state)
809  * @lock: queue lock
810  * @stuck_timer: timer that fires if queue gets stuck
811  * @trans: pointer back to transport (for timer)
812  * @need_update: indicates need to update read/write index
813  * @ampdu: true if this queue is an ampdu queue for an specific RA/TID
814  * @wd_timeout: queue watchdog timeout (jiffies) - per queue
815  * @frozen: tx stuck queue timer is frozen
816  * @frozen_expiry_remainder: remember how long until the timer fires
817  * @bc_tbl: byte count table of the queue (relevant only for gen2 transport)
818  * @write_ptr: 1-st empty entry (index) host_w
819  * @read_ptr: last used entry (index) host_r
820  * @dma_addr:  physical addr for BD's
821  * @n_window: safe queue window
822  * @id: queue id
823  * @low_mark: low watermark, resume queue if free space more than this
824  * @high_mark: high watermark, stop queue if free space less than this
825  *
826  * A Tx queue consists of circular buffer of BDs (a.k.a. TFDs, transmit frame
827  * descriptors) and required locking structures.
828  *
829  * Note the difference between TFD_QUEUE_SIZE_MAX and n_window: the hardware
830  * always assumes 256 descriptors, so TFD_QUEUE_SIZE_MAX is always 256 (unless
831  * there might be HW changes in the future). For the normal TX
832  * queues, n_window, which is the size of the software queue data
833  * is also 256; however, for the command queue, n_window is only
834  * 32 since we don't need so many commands pending. Since the HW
835  * still uses 256 BDs for DMA though, TFD_QUEUE_SIZE_MAX stays 256.
836  * This means that we end up with the following:
837  *  HW entries: | 0 | ... | N * 32 | ... | N * 32 + 31 | ... | 255 |
838  *  SW entries:           | 0      | ... | 31          |
839  * where N is a number between 0 and 7. This means that the SW
840  * data is a window overlayed over the HW queue.
841  */
842 struct iwl_txq {
843 	void *tfds;
844 	struct iwl_pcie_first_tb_buf *first_tb_bufs;
845 	dma_addr_t first_tb_dma;
846 	struct iwl_pcie_txq_entry *entries;
847 	/* lock for syncing changes on the queue */
848 	spinlock_t lock;
849 	unsigned long frozen_expiry_remainder;
850 	struct timer_list stuck_timer;
851 	struct iwl_trans *trans;
852 	bool need_update;
853 	bool frozen;
854 	bool ampdu;
855 	int block;
856 	unsigned long wd_timeout;
857 	struct sk_buff_head overflow_q;
858 	struct iwl_dma_ptr bc_tbl;
859 
860 	int write_ptr;
861 	int read_ptr;
862 	dma_addr_t dma_addr;
863 	int n_window;
864 	u32 id;
865 	int low_mark;
866 	int high_mark;
867 
868 	bool overflow_tx;
869 };
870 
871 /**
872  * struct iwl_trans_txqs - transport tx queues data
873  *
874  * @bc_table_dword: true if the BC table expects DWORD (as opposed to bytes)
875  * @page_offs: offset from skb->cb to mac header page pointer
876  * @dev_cmd_offs: offset from skb->cb to iwl_device_tx_cmd pointer
877  * @queue_used - bit mask of used queues
878  * @queue_stopped - bit mask of stopped queues
879  * @scd_bc_tbls: gen1 pointer to the byte count table of the scheduler
880  */
881 struct iwl_trans_txqs {
882 	unsigned long queue_used[BITS_TO_LONGS(IWL_MAX_TVQM_QUEUES)];
883 	unsigned long queue_stopped[BITS_TO_LONGS(IWL_MAX_TVQM_QUEUES)];
884 	struct iwl_txq *txq[IWL_MAX_TVQM_QUEUES];
885 	struct dma_pool *bc_pool;
886 	size_t bc_tbl_size;
887 	bool bc_table_dword;
888 	u8 page_offs;
889 	u8 dev_cmd_offs;
890 	struct iwl_tso_hdr_page __percpu *tso_hdr_page;
891 
892 	struct {
893 		u8 fifo;
894 		u8 q_id;
895 		unsigned int wdg_timeout;
896 	} cmd;
897 
898 	struct {
899 		u8 max_tbs;
900 		u16 size;
901 		u8 addr_size;
902 	} tfd;
903 
904 	struct iwl_dma_ptr scd_bc_tbls;
905 };
906 
907 /**
908  * struct iwl_trans - transport common data
909  *
910  * @ops - pointer to iwl_trans_ops
911  * @op_mode - pointer to the op_mode
912  * @trans_cfg: the trans-specific configuration part
913  * @cfg - pointer to the configuration
914  * @drv - pointer to iwl_drv
915  * @status: a bit-mask of transport status flags
916  * @dev - pointer to struct device * that represents the device
917  * @max_skb_frags: maximum number of fragments an SKB can have when transmitted.
918  *	0 indicates that frag SKBs (NETIF_F_SG) aren't supported.
919  * @hw_rf_id a u32 with the device RF ID
920  * @hw_id: a u32 with the ID of the device / sub-device.
921  *	Set during transport allocation.
922  * @hw_id_str: a string with info about HW ID. Set during transport allocation.
923  * @pm_support: set to true in start_hw if link pm is supported
924  * @ltr_enabled: set to true if the LTR is enabled
925  * @wide_cmd_header: true when ucode supports wide command header format
926  * @wait_command_queue: wait queue for sync commands
927  * @num_rx_queues: number of RX queues allocated by the transport;
928  *	the transport must set this before calling iwl_drv_start()
929  * @iml_len: the length of the image loader
930  * @iml: a pointer to the image loader itself
931  * @dev_cmd_pool: pool for Tx cmd allocation - for internal use only.
932  *	The user should use iwl_trans_{alloc,free}_tx_cmd.
933  * @rx_mpdu_cmd: MPDU RX command ID, must be assigned by opmode before
934  *	starting the firmware, used for tracing
935  * @rx_mpdu_cmd_hdr_size: used for tracing, amount of data before the
936  *	start of the 802.11 header in the @rx_mpdu_cmd
937  * @dflt_pwr_limit: default power limit fetched from the platform (ACPI)
938  * @system_pm_mode: the system-wide power management mode in use.
939  *	This mode is set dynamically, depending on the WoWLAN values
940  *	configured from the userspace at runtime.
941  * @iwl_trans_txqs: transport tx queues data.
942  */
943 struct iwl_trans {
944 	const struct iwl_trans_ops *ops;
945 	struct iwl_op_mode *op_mode;
946 	const struct iwl_cfg_trans_params *trans_cfg;
947 	const struct iwl_cfg *cfg;
948 	struct iwl_drv *drv;
949 	enum iwl_trans_state state;
950 	unsigned long status;
951 
952 	struct device *dev;
953 	u32 max_skb_frags;
954 	u32 hw_rev;
955 	u32 hw_rf_id;
956 	u32 hw_id;
957 	char hw_id_str[52];
958 	u32 sku_id[3];
959 
960 	u8 rx_mpdu_cmd, rx_mpdu_cmd_hdr_size;
961 
962 	bool pm_support;
963 	bool ltr_enabled;
964 	u8 pnvm_loaded:1;
965 	u8 reduce_power_loaded:1;
966 
967 	const struct iwl_hcmd_arr *command_groups;
968 	int command_groups_size;
969 	bool wide_cmd_header;
970 
971 	wait_queue_head_t wait_command_queue;
972 	u8 num_rx_queues;
973 
974 	size_t iml_len;
975 	u8 *iml;
976 
977 	/* The following fields are internal only */
978 	struct kmem_cache *dev_cmd_pool;
979 	char dev_cmd_pool_name[50];
980 
981 	struct dentry *dbgfs_dir;
982 
983 #ifdef CONFIG_LOCKDEP
984 	struct lockdep_map sync_cmd_lockdep_map;
985 #endif
986 
987 	struct iwl_trans_debug dbg;
988 	struct iwl_self_init_dram init_dram;
989 
990 	enum iwl_plat_pm_mode system_pm_mode;
991 
992 	const char *name;
993 	struct iwl_trans_txqs txqs;
994 
995 	/* pointer to trans specific struct */
996 	/*Ensure that this pointer will always be aligned to sizeof pointer */
997 	char trans_specific[] __aligned(sizeof(void *));
998 };
999 
1000 const char *iwl_get_cmd_string(struct iwl_trans *trans, u32 id);
1001 int iwl_cmd_groups_verify_sorted(const struct iwl_trans_config *trans);
1002 
1003 static inline void iwl_trans_configure(struct iwl_trans *trans,
1004 				       const struct iwl_trans_config *trans_cfg)
1005 {
1006 	trans->op_mode = trans_cfg->op_mode;
1007 
1008 	trans->ops->configure(trans, trans_cfg);
1009 	WARN_ON(iwl_cmd_groups_verify_sorted(trans_cfg));
1010 }
1011 
1012 static inline int iwl_trans_start_hw(struct iwl_trans *trans)
1013 {
1014 	might_sleep();
1015 
1016 	return trans->ops->start_hw(trans);
1017 }
1018 
1019 static inline void iwl_trans_op_mode_leave(struct iwl_trans *trans)
1020 {
1021 	might_sleep();
1022 
1023 	if (trans->ops->op_mode_leave)
1024 		trans->ops->op_mode_leave(trans);
1025 
1026 	trans->op_mode = NULL;
1027 
1028 	trans->state = IWL_TRANS_NO_FW;
1029 }
1030 
1031 static inline void iwl_trans_fw_alive(struct iwl_trans *trans, u32 scd_addr)
1032 {
1033 	might_sleep();
1034 
1035 	trans->state = IWL_TRANS_FW_ALIVE;
1036 
1037 	trans->ops->fw_alive(trans, scd_addr);
1038 }
1039 
1040 static inline int iwl_trans_start_fw(struct iwl_trans *trans,
1041 				     const struct fw_img *fw,
1042 				     bool run_in_rfkill)
1043 {
1044 	int ret;
1045 
1046 	might_sleep();
1047 
1048 	WARN_ON_ONCE(!trans->rx_mpdu_cmd);
1049 
1050 	clear_bit(STATUS_FW_ERROR, &trans->status);
1051 	ret = trans->ops->start_fw(trans, fw, run_in_rfkill);
1052 	if (ret == 0)
1053 		trans->state = IWL_TRANS_FW_STARTED;
1054 
1055 	return ret;
1056 }
1057 
1058 static inline void iwl_trans_stop_device(struct iwl_trans *trans)
1059 {
1060 	might_sleep();
1061 
1062 	trans->ops->stop_device(trans);
1063 
1064 	trans->state = IWL_TRANS_NO_FW;
1065 }
1066 
1067 static inline int iwl_trans_d3_suspend(struct iwl_trans *trans, bool test,
1068 				       bool reset)
1069 {
1070 	might_sleep();
1071 	if (!trans->ops->d3_suspend)
1072 		return 0;
1073 
1074 	return trans->ops->d3_suspend(trans, test, reset);
1075 }
1076 
1077 static inline int iwl_trans_d3_resume(struct iwl_trans *trans,
1078 				      enum iwl_d3_status *status,
1079 				      bool test, bool reset)
1080 {
1081 	might_sleep();
1082 	if (!trans->ops->d3_resume)
1083 		return 0;
1084 
1085 	return trans->ops->d3_resume(trans, status, test, reset);
1086 }
1087 
1088 static inline struct iwl_trans_dump_data *
1089 iwl_trans_dump_data(struct iwl_trans *trans, u32 dump_mask)
1090 {
1091 	if (!trans->ops->dump_data)
1092 		return NULL;
1093 	return trans->ops->dump_data(trans, dump_mask);
1094 }
1095 
1096 static inline struct iwl_device_tx_cmd *
1097 iwl_trans_alloc_tx_cmd(struct iwl_trans *trans)
1098 {
1099 	return kmem_cache_zalloc(trans->dev_cmd_pool, GFP_ATOMIC);
1100 }
1101 
1102 int iwl_trans_send_cmd(struct iwl_trans *trans, struct iwl_host_cmd *cmd);
1103 
1104 static inline void iwl_trans_free_tx_cmd(struct iwl_trans *trans,
1105 					 struct iwl_device_tx_cmd *dev_cmd)
1106 {
1107 	kmem_cache_free(trans->dev_cmd_pool, dev_cmd);
1108 }
1109 
1110 static inline int iwl_trans_tx(struct iwl_trans *trans, struct sk_buff *skb,
1111 			       struct iwl_device_tx_cmd *dev_cmd, int queue)
1112 {
1113 	if (unlikely(test_bit(STATUS_FW_ERROR, &trans->status)))
1114 		return -EIO;
1115 
1116 	if (WARN_ON_ONCE(trans->state != IWL_TRANS_FW_ALIVE)) {
1117 		IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state);
1118 		return -EIO;
1119 	}
1120 
1121 	return trans->ops->tx(trans, skb, dev_cmd, queue);
1122 }
1123 
1124 static inline void iwl_trans_reclaim(struct iwl_trans *trans, int queue,
1125 				     int ssn, struct sk_buff_head *skbs)
1126 {
1127 	if (WARN_ON_ONCE(trans->state != IWL_TRANS_FW_ALIVE)) {
1128 		IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state);
1129 		return;
1130 	}
1131 
1132 	trans->ops->reclaim(trans, queue, ssn, skbs);
1133 }
1134 
1135 static inline void iwl_trans_set_q_ptrs(struct iwl_trans *trans, int queue,
1136 					int ptr)
1137 {
1138 	if (WARN_ON_ONCE(trans->state != IWL_TRANS_FW_ALIVE)) {
1139 		IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state);
1140 		return;
1141 	}
1142 
1143 	trans->ops->set_q_ptrs(trans, queue, ptr);
1144 }
1145 
1146 static inline void iwl_trans_txq_disable(struct iwl_trans *trans, int queue,
1147 					 bool configure_scd)
1148 {
1149 	trans->ops->txq_disable(trans, queue, configure_scd);
1150 }
1151 
1152 static inline bool
1153 iwl_trans_txq_enable_cfg(struct iwl_trans *trans, int queue, u16 ssn,
1154 			 const struct iwl_trans_txq_scd_cfg *cfg,
1155 			 unsigned int queue_wdg_timeout)
1156 {
1157 	might_sleep();
1158 
1159 	if (WARN_ON_ONCE(trans->state != IWL_TRANS_FW_ALIVE)) {
1160 		IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state);
1161 		return false;
1162 	}
1163 
1164 	return trans->ops->txq_enable(trans, queue, ssn,
1165 				      cfg, queue_wdg_timeout);
1166 }
1167 
1168 static inline int
1169 iwl_trans_get_rxq_dma_data(struct iwl_trans *trans, int queue,
1170 			   struct iwl_trans_rxq_dma_data *data)
1171 {
1172 	if (WARN_ON_ONCE(!trans->ops->rxq_dma_data))
1173 		return -ENOTSUPP;
1174 
1175 	return trans->ops->rxq_dma_data(trans, queue, data);
1176 }
1177 
1178 static inline void
1179 iwl_trans_txq_free(struct iwl_trans *trans, int queue)
1180 {
1181 	if (WARN_ON_ONCE(!trans->ops->txq_free))
1182 		return;
1183 
1184 	trans->ops->txq_free(trans, queue);
1185 }
1186 
1187 static inline int
1188 iwl_trans_txq_alloc(struct iwl_trans *trans,
1189 		    __le16 flags, u8 sta_id, u8 tid,
1190 		    int cmd_id, int size,
1191 		    unsigned int wdg_timeout)
1192 {
1193 	might_sleep();
1194 
1195 	if (WARN_ON_ONCE(!trans->ops->txq_alloc))
1196 		return -ENOTSUPP;
1197 
1198 	if (WARN_ON_ONCE(trans->state != IWL_TRANS_FW_ALIVE)) {
1199 		IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state);
1200 		return -EIO;
1201 	}
1202 
1203 	return trans->ops->txq_alloc(trans, flags, sta_id, tid,
1204 				     cmd_id, size, wdg_timeout);
1205 }
1206 
1207 static inline void iwl_trans_txq_set_shared_mode(struct iwl_trans *trans,
1208 						 int queue, bool shared_mode)
1209 {
1210 	if (trans->ops->txq_set_shared_mode)
1211 		trans->ops->txq_set_shared_mode(trans, queue, shared_mode);
1212 }
1213 
1214 static inline void iwl_trans_txq_enable(struct iwl_trans *trans, int queue,
1215 					int fifo, int sta_id, int tid,
1216 					int frame_limit, u16 ssn,
1217 					unsigned int queue_wdg_timeout)
1218 {
1219 	struct iwl_trans_txq_scd_cfg cfg = {
1220 		.fifo = fifo,
1221 		.sta_id = sta_id,
1222 		.tid = tid,
1223 		.frame_limit = frame_limit,
1224 		.aggregate = sta_id >= 0,
1225 	};
1226 
1227 	iwl_trans_txq_enable_cfg(trans, queue, ssn, &cfg, queue_wdg_timeout);
1228 }
1229 
1230 static inline
1231 void iwl_trans_ac_txq_enable(struct iwl_trans *trans, int queue, int fifo,
1232 			     unsigned int queue_wdg_timeout)
1233 {
1234 	struct iwl_trans_txq_scd_cfg cfg = {
1235 		.fifo = fifo,
1236 		.sta_id = -1,
1237 		.tid = IWL_MAX_TID_COUNT,
1238 		.frame_limit = IWL_FRAME_LIMIT,
1239 		.aggregate = false,
1240 	};
1241 
1242 	iwl_trans_txq_enable_cfg(trans, queue, 0, &cfg, queue_wdg_timeout);
1243 }
1244 
1245 static inline void iwl_trans_freeze_txq_timer(struct iwl_trans *trans,
1246 					      unsigned long txqs,
1247 					      bool freeze)
1248 {
1249 	if (WARN_ON_ONCE(trans->state != IWL_TRANS_FW_ALIVE)) {
1250 		IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state);
1251 		return;
1252 	}
1253 
1254 	if (trans->ops->freeze_txq_timer)
1255 		trans->ops->freeze_txq_timer(trans, txqs, freeze);
1256 }
1257 
1258 static inline void iwl_trans_block_txq_ptrs(struct iwl_trans *trans,
1259 					    bool block)
1260 {
1261 	if (WARN_ON_ONCE(trans->state != IWL_TRANS_FW_ALIVE)) {
1262 		IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state);
1263 		return;
1264 	}
1265 
1266 	if (trans->ops->block_txq_ptrs)
1267 		trans->ops->block_txq_ptrs(trans, block);
1268 }
1269 
1270 static inline int iwl_trans_wait_tx_queues_empty(struct iwl_trans *trans,
1271 						 u32 txqs)
1272 {
1273 	if (WARN_ON_ONCE(!trans->ops->wait_tx_queues_empty))
1274 		return -ENOTSUPP;
1275 
1276 	/* No need to wait if the firmware is not alive */
1277 	if (trans->state != IWL_TRANS_FW_ALIVE) {
1278 		IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state);
1279 		return -EIO;
1280 	}
1281 
1282 	return trans->ops->wait_tx_queues_empty(trans, txqs);
1283 }
1284 
1285 static inline int iwl_trans_wait_txq_empty(struct iwl_trans *trans, int queue)
1286 {
1287 	if (WARN_ON_ONCE(!trans->ops->wait_txq_empty))
1288 		return -ENOTSUPP;
1289 
1290 	if (WARN_ON_ONCE(trans->state != IWL_TRANS_FW_ALIVE)) {
1291 		IWL_ERR(trans, "%s bad state = %d\n", __func__, trans->state);
1292 		return -EIO;
1293 	}
1294 
1295 	return trans->ops->wait_txq_empty(trans, queue);
1296 }
1297 
1298 static inline void iwl_trans_write8(struct iwl_trans *trans, u32 ofs, u8 val)
1299 {
1300 	trans->ops->write8(trans, ofs, val);
1301 }
1302 
1303 static inline void iwl_trans_write32(struct iwl_trans *trans, u32 ofs, u32 val)
1304 {
1305 	trans->ops->write32(trans, ofs, val);
1306 }
1307 
1308 static inline u32 iwl_trans_read32(struct iwl_trans *trans, u32 ofs)
1309 {
1310 	return trans->ops->read32(trans, ofs);
1311 }
1312 
1313 static inline u32 iwl_trans_read_prph(struct iwl_trans *trans, u32 ofs)
1314 {
1315 	return trans->ops->read_prph(trans, ofs);
1316 }
1317 
1318 static inline void iwl_trans_write_prph(struct iwl_trans *trans, u32 ofs,
1319 					u32 val)
1320 {
1321 	return trans->ops->write_prph(trans, ofs, val);
1322 }
1323 
1324 static inline int iwl_trans_read_mem(struct iwl_trans *trans, u32 addr,
1325 				     void *buf, int dwords)
1326 {
1327 	return trans->ops->read_mem(trans, addr, buf, dwords);
1328 }
1329 
1330 #define iwl_trans_read_mem_bytes(trans, addr, buf, bufsize)		      \
1331 	do {								      \
1332 		if (__builtin_constant_p(bufsize))			      \
1333 			BUILD_BUG_ON((bufsize) % sizeof(u32));		      \
1334 		iwl_trans_read_mem(trans, addr, buf, (bufsize) / sizeof(u32));\
1335 	} while (0)
1336 
1337 static inline u32 iwl_trans_read_mem32(struct iwl_trans *trans, u32 addr)
1338 {
1339 	u32 value;
1340 
1341 	if (WARN_ON(iwl_trans_read_mem(trans, addr, &value, 1)))
1342 		return 0xa5a5a5a5;
1343 
1344 	return value;
1345 }
1346 
1347 static inline int iwl_trans_write_mem(struct iwl_trans *trans, u32 addr,
1348 				      const void *buf, int dwords)
1349 {
1350 	return trans->ops->write_mem(trans, addr, buf, dwords);
1351 }
1352 
1353 static inline u32 iwl_trans_write_mem32(struct iwl_trans *trans, u32 addr,
1354 					u32 val)
1355 {
1356 	return iwl_trans_write_mem(trans, addr, &val, 1);
1357 }
1358 
1359 static inline void iwl_trans_set_pmi(struct iwl_trans *trans, bool state)
1360 {
1361 	if (trans->ops->set_pmi)
1362 		trans->ops->set_pmi(trans, state);
1363 }
1364 
1365 static inline void iwl_trans_sw_reset(struct iwl_trans *trans)
1366 {
1367 	if (trans->ops->sw_reset)
1368 		trans->ops->sw_reset(trans);
1369 }
1370 
1371 static inline void
1372 iwl_trans_set_bits_mask(struct iwl_trans *trans, u32 reg, u32 mask, u32 value)
1373 {
1374 	trans->ops->set_bits_mask(trans, reg, mask, value);
1375 }
1376 
1377 #define iwl_trans_grab_nic_access(trans)		\
1378 	__cond_lock(nic_access,				\
1379 		    likely((trans)->ops->grab_nic_access(trans)))
1380 
1381 static inline void __releases(nic_access)
1382 iwl_trans_release_nic_access(struct iwl_trans *trans)
1383 {
1384 	trans->ops->release_nic_access(trans);
1385 	__release(nic_access);
1386 }
1387 
1388 static inline void iwl_trans_fw_error(struct iwl_trans *trans, bool sync)
1389 {
1390 	if (WARN_ON_ONCE(!trans->op_mode))
1391 		return;
1392 
1393 	/* prevent double restarts due to the same erroneous FW */
1394 	if (!test_and_set_bit(STATUS_FW_ERROR, &trans->status)) {
1395 		iwl_op_mode_nic_error(trans->op_mode, sync);
1396 		trans->state = IWL_TRANS_NO_FW;
1397 	}
1398 }
1399 
1400 static inline bool iwl_trans_fw_running(struct iwl_trans *trans)
1401 {
1402 	return trans->state == IWL_TRANS_FW_ALIVE;
1403 }
1404 
1405 static inline void iwl_trans_sync_nmi(struct iwl_trans *trans)
1406 {
1407 	if (trans->ops->sync_nmi)
1408 		trans->ops->sync_nmi(trans);
1409 }
1410 
1411 void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,
1412 				  u32 sw_err_bit);
1413 
1414 static inline int iwl_trans_set_pnvm(struct iwl_trans *trans,
1415 				     const void *data, u32 len)
1416 {
1417 	if (trans->ops->set_pnvm) {
1418 		int ret = trans->ops->set_pnvm(trans, data, len);
1419 
1420 		if (ret)
1421 			return ret;
1422 	}
1423 
1424 	trans->pnvm_loaded = true;
1425 
1426 	return 0;
1427 }
1428 
1429 static inline int iwl_trans_set_reduce_power(struct iwl_trans *trans,
1430 					     const void *data, u32 len)
1431 {
1432 	if (trans->ops->set_reduce_power) {
1433 		int ret = trans->ops->set_reduce_power(trans, data, len);
1434 
1435 		if (ret)
1436 			return ret;
1437 	}
1438 
1439 	trans->reduce_power_loaded = true;
1440 	return 0;
1441 }
1442 
1443 static inline bool iwl_trans_dbg_ini_valid(struct iwl_trans *trans)
1444 {
1445 	return trans->dbg.internal_ini_cfg != IWL_INI_CFG_STATE_NOT_LOADED ||
1446 		trans->dbg.external_ini_cfg != IWL_INI_CFG_STATE_NOT_LOADED;
1447 }
1448 
1449 static inline void iwl_trans_interrupts(struct iwl_trans *trans, bool enable)
1450 {
1451 	if (trans->ops->interrupts)
1452 		trans->ops->interrupts(trans, enable);
1453 }
1454 
1455 /*****************************************************
1456  * transport helper functions
1457  *****************************************************/
1458 struct iwl_trans *iwl_trans_alloc(unsigned int priv_size,
1459 			  struct device *dev,
1460 			  const struct iwl_trans_ops *ops,
1461 			  const struct iwl_cfg_trans_params *cfg_trans);
1462 int iwl_trans_init(struct iwl_trans *trans);
1463 void iwl_trans_free(struct iwl_trans *trans);
1464 
1465 /*****************************************************
1466 * driver (transport) register/unregister functions
1467 ******************************************************/
1468 int __must_check iwl_pci_register_driver(void);
1469 void iwl_pci_unregister_driver(void);
1470 
1471 #endif /* __iwl_trans_h__ */
1472