1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /*
3  * Copyright (C) 2007-2015, 2018-2022 Intel Corporation
4  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
5  * Copyright (C) 2016-2017 Intel Deutschland GmbH
6  */
7 #include <linux/pci.h>
8 #include <linux/interrupt.h>
9 #include <linux/debugfs.h>
10 #include <linux/sched.h>
11 #include <linux/bitops.h>
12 #include <linux/gfp.h>
13 #include <linux/vmalloc.h>
14 #include <linux/module.h>
15 #include <linux/wait.h>
16 #include <linux/seq_file.h>
17 
18 #include "iwl-drv.h"
19 #include "iwl-trans.h"
20 #include "iwl-csr.h"
21 #include "iwl-prph.h"
22 #include "iwl-scd.h"
23 #include "iwl-agn-hw.h"
24 #include "fw/error-dump.h"
25 #include "fw/dbg.h"
26 #include "fw/api/tx.h"
27 #include "mei/iwl-mei.h"
28 #include "internal.h"
29 #include "iwl-fh.h"
30 #include "iwl-context-info-gen3.h"
31 
32 /* extended range in FW SRAM */
33 #define IWL_FW_MEM_EXTENDED_START	0x40000
34 #define IWL_FW_MEM_EXTENDED_END		0x57FFF
35 
36 void iwl_trans_pcie_dump_regs(struct iwl_trans *trans)
37 {
38 #define PCI_DUMP_SIZE		352
39 #define PCI_MEM_DUMP_SIZE	64
40 #define PCI_PARENT_DUMP_SIZE	524
41 #define PREFIX_LEN		32
42 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
43 	struct pci_dev *pdev = trans_pcie->pci_dev;
44 	u32 i, pos, alloc_size, *ptr, *buf;
45 	char *prefix;
46 
47 	if (trans_pcie->pcie_dbg_dumped_once)
48 		return;
49 
50 	/* Should be a multiple of 4 */
51 	BUILD_BUG_ON(PCI_DUMP_SIZE > 4096 || PCI_DUMP_SIZE & 0x3);
52 	BUILD_BUG_ON(PCI_MEM_DUMP_SIZE > 4096 || PCI_MEM_DUMP_SIZE & 0x3);
53 	BUILD_BUG_ON(PCI_PARENT_DUMP_SIZE > 4096 || PCI_PARENT_DUMP_SIZE & 0x3);
54 
55 	/* Alloc a max size buffer */
56 	alloc_size = PCI_ERR_ROOT_ERR_SRC +  4 + PREFIX_LEN;
57 	alloc_size = max_t(u32, alloc_size, PCI_DUMP_SIZE + PREFIX_LEN);
58 	alloc_size = max_t(u32, alloc_size, PCI_MEM_DUMP_SIZE + PREFIX_LEN);
59 	alloc_size = max_t(u32, alloc_size, PCI_PARENT_DUMP_SIZE + PREFIX_LEN);
60 
61 	buf = kmalloc(alloc_size, GFP_ATOMIC);
62 	if (!buf)
63 		return;
64 	prefix = (char *)buf + alloc_size - PREFIX_LEN;
65 
66 	IWL_ERR(trans, "iwlwifi transaction failed, dumping registers\n");
67 
68 	/* Print wifi device registers */
69 	sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
70 	IWL_ERR(trans, "iwlwifi device config registers:\n");
71 	for (i = 0, ptr = buf; i < PCI_DUMP_SIZE; i += 4, ptr++)
72 		if (pci_read_config_dword(pdev, i, ptr))
73 			goto err_read;
74 	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
75 
76 	IWL_ERR(trans, "iwlwifi device memory mapped registers:\n");
77 	for (i = 0, ptr = buf; i < PCI_MEM_DUMP_SIZE; i += 4, ptr++)
78 		*ptr = iwl_read32(trans, i);
79 	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
80 
81 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
82 	if (pos) {
83 		IWL_ERR(trans, "iwlwifi device AER capability structure:\n");
84 		for (i = 0, ptr = buf; i < PCI_ERR_ROOT_COMMAND; i += 4, ptr++)
85 			if (pci_read_config_dword(pdev, pos + i, ptr))
86 				goto err_read;
87 		print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET,
88 			       32, 4, buf, i, 0);
89 	}
90 
91 	/* Print parent device registers next */
92 	if (!pdev->bus->self)
93 		goto out;
94 
95 	pdev = pdev->bus->self;
96 	sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
97 
98 	IWL_ERR(trans, "iwlwifi parent port (%s) config registers:\n",
99 		pci_name(pdev));
100 	for (i = 0, ptr = buf; i < PCI_PARENT_DUMP_SIZE; i += 4, ptr++)
101 		if (pci_read_config_dword(pdev, i, ptr))
102 			goto err_read;
103 	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
104 
105 	/* Print root port AER registers */
106 	pos = 0;
107 	pdev = pcie_find_root_port(pdev);
108 	if (pdev)
109 		pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
110 	if (pos) {
111 		IWL_ERR(trans, "iwlwifi root port (%s) AER cap structure:\n",
112 			pci_name(pdev));
113 		sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
114 		for (i = 0, ptr = buf; i <= PCI_ERR_ROOT_ERR_SRC; i += 4, ptr++)
115 			if (pci_read_config_dword(pdev, pos + i, ptr))
116 				goto err_read;
117 		print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32,
118 			       4, buf, i, 0);
119 	}
120 	goto out;
121 
122 err_read:
123 	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
124 	IWL_ERR(trans, "Read failed at 0x%X\n", i);
125 out:
126 	trans_pcie->pcie_dbg_dumped_once = 1;
127 	kfree(buf);
128 }
129 
130 static int iwl_trans_pcie_sw_reset(struct iwl_trans *trans,
131 				   bool retake_ownership)
132 {
133 	/* Reset entire device - do controller reset (results in SHRD_HW_RST) */
134 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
135 		iwl_set_bit(trans, CSR_GP_CNTRL,
136 			    CSR_GP_CNTRL_REG_FLAG_SW_RESET);
137 	else
138 		iwl_set_bit(trans, CSR_RESET,
139 			    CSR_RESET_REG_FLAG_SW_RESET);
140 	usleep_range(5000, 6000);
141 
142 	if (retake_ownership)
143 		return iwl_pcie_prepare_card_hw(trans);
144 
145 	return 0;
146 }
147 
148 static void iwl_pcie_free_fw_monitor(struct iwl_trans *trans)
149 {
150 	struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
151 
152 	if (!fw_mon->size)
153 		return;
154 
155 	dma_free_coherent(trans->dev, fw_mon->size, fw_mon->block,
156 			  fw_mon->physical);
157 
158 	fw_mon->block = NULL;
159 	fw_mon->physical = 0;
160 	fw_mon->size = 0;
161 }
162 
163 static void iwl_pcie_alloc_fw_monitor_block(struct iwl_trans *trans,
164 					    u8 max_power, u8 min_power)
165 {
166 	struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
167 	void *block = NULL;
168 	dma_addr_t physical = 0;
169 	u32 size = 0;
170 	u8 power;
171 
172 	if (fw_mon->size)
173 		return;
174 
175 	for (power = max_power; power >= min_power; power--) {
176 		size = BIT(power);
177 		block = dma_alloc_coherent(trans->dev, size, &physical,
178 					   GFP_KERNEL | __GFP_NOWARN);
179 		if (!block)
180 			continue;
181 
182 		IWL_INFO(trans,
183 			 "Allocated 0x%08x bytes for firmware monitor.\n",
184 			 size);
185 		break;
186 	}
187 
188 	if (WARN_ON_ONCE(!block))
189 		return;
190 
191 	if (power != max_power)
192 		IWL_ERR(trans,
193 			"Sorry - debug buffer is only %luK while you requested %luK\n",
194 			(unsigned long)BIT(power - 10),
195 			(unsigned long)BIT(max_power - 10));
196 
197 	fw_mon->block = block;
198 	fw_mon->physical = physical;
199 	fw_mon->size = size;
200 }
201 
202 void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power)
203 {
204 	if (!max_power) {
205 		/* default max_power is maximum */
206 		max_power = 26;
207 	} else {
208 		max_power += 11;
209 	}
210 
211 	if (WARN(max_power > 26,
212 		 "External buffer size for monitor is too big %d, check the FW TLV\n",
213 		 max_power))
214 		return;
215 
216 	if (trans->dbg.fw_mon.size)
217 		return;
218 
219 	iwl_pcie_alloc_fw_monitor_block(trans, max_power, 11);
220 }
221 
222 static u32 iwl_trans_pcie_read_shr(struct iwl_trans *trans, u32 reg)
223 {
224 	iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
225 		    ((reg & 0x0000ffff) | (2 << 28)));
226 	return iwl_read32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG);
227 }
228 
229 static void iwl_trans_pcie_write_shr(struct iwl_trans *trans, u32 reg, u32 val)
230 {
231 	iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG, val);
232 	iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
233 		    ((reg & 0x0000ffff) | (3 << 28)));
234 }
235 
236 static void iwl_pcie_set_pwr(struct iwl_trans *trans, bool vaux)
237 {
238 	if (trans->cfg->apmg_not_supported)
239 		return;
240 
241 	if (vaux && pci_pme_capable(to_pci_dev(trans->dev), PCI_D3cold))
242 		iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
243 				       APMG_PS_CTRL_VAL_PWR_SRC_VAUX,
244 				       ~APMG_PS_CTRL_MSK_PWR_SRC);
245 	else
246 		iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
247 				       APMG_PS_CTRL_VAL_PWR_SRC_VMAIN,
248 				       ~APMG_PS_CTRL_MSK_PWR_SRC);
249 }
250 
251 /* PCI registers */
252 #define PCI_CFG_RETRY_TIMEOUT	0x041
253 
254 void iwl_pcie_apm_config(struct iwl_trans *trans)
255 {
256 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
257 	u16 lctl;
258 	u16 cap;
259 
260 	/*
261 	 * L0S states have been found to be unstable with our devices
262 	 * and in newer hardware they are not officially supported at
263 	 * all, so we must always set the L0S_DISABLED bit.
264 	 */
265 	iwl_set_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_DISABLED);
266 
267 	pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_LNKCTL, &lctl);
268 	trans->pm_support = !(lctl & PCI_EXP_LNKCTL_ASPM_L0S);
269 
270 	pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_DEVCTL2, &cap);
271 	trans->ltr_enabled = cap & PCI_EXP_DEVCTL2_LTR_EN;
272 	IWL_DEBUG_POWER(trans, "L1 %sabled - LTR %sabled\n",
273 			(lctl & PCI_EXP_LNKCTL_ASPM_L1) ? "En" : "Dis",
274 			trans->ltr_enabled ? "En" : "Dis");
275 }
276 
277 /*
278  * Start up NIC's basic functionality after it has been reset
279  * (e.g. after platform boot, or shutdown via iwl_pcie_apm_stop())
280  * NOTE:  This does not load uCode nor start the embedded processor
281  */
282 static int iwl_pcie_apm_init(struct iwl_trans *trans)
283 {
284 	int ret;
285 
286 	IWL_DEBUG_INFO(trans, "Init card's basic functions\n");
287 
288 	/*
289 	 * Use "set_bit" below rather than "write", to preserve any hardware
290 	 * bits already set by default after reset.
291 	 */
292 
293 	/* Disable L0S exit timer (platform NMI Work/Around) */
294 	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000)
295 		iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
296 			    CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER);
297 
298 	/*
299 	 * Disable L0s without affecting L1;
300 	 *  don't wait for ICH L0s (ICH bug W/A)
301 	 */
302 	iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
303 		    CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX);
304 
305 	/* Set FH wait threshold to maximum (HW error during stress W/A) */
306 	iwl_set_bit(trans, CSR_DBG_HPET_MEM_REG, CSR_DBG_HPET_MEM_REG_VAL);
307 
308 	/*
309 	 * Enable HAP INTA (interrupt from management bus) to
310 	 * wake device's PCI Express link L1a -> L0s
311 	 */
312 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
313 		    CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
314 
315 	iwl_pcie_apm_config(trans);
316 
317 	/* Configure analog phase-lock-loop before activating to D0A */
318 	if (trans->trans_cfg->base_params->pll_cfg)
319 		iwl_set_bit(trans, CSR_ANA_PLL_CFG, CSR50_ANA_PLL_CFG_VAL);
320 
321 	ret = iwl_finish_nic_init(trans);
322 	if (ret)
323 		return ret;
324 
325 	if (trans->cfg->host_interrupt_operation_mode) {
326 		/*
327 		 * This is a bit of an abuse - This is needed for 7260 / 3160
328 		 * only check host_interrupt_operation_mode even if this is
329 		 * not related to host_interrupt_operation_mode.
330 		 *
331 		 * Enable the oscillator to count wake up time for L1 exit. This
332 		 * consumes slightly more power (100uA) - but allows to be sure
333 		 * that we wake up from L1 on time.
334 		 *
335 		 * This looks weird: read twice the same register, discard the
336 		 * value, set a bit, and yet again, read that same register
337 		 * just to discard the value. But that's the way the hardware
338 		 * seems to like it.
339 		 */
340 		iwl_read_prph(trans, OSC_CLK);
341 		iwl_read_prph(trans, OSC_CLK);
342 		iwl_set_bits_prph(trans, OSC_CLK, OSC_CLK_FORCE_CONTROL);
343 		iwl_read_prph(trans, OSC_CLK);
344 		iwl_read_prph(trans, OSC_CLK);
345 	}
346 
347 	/*
348 	 * Enable DMA clock and wait for it to stabilize.
349 	 *
350 	 * Write to "CLK_EN_REG"; "1" bits enable clocks, while "0"
351 	 * bits do not disable clocks.  This preserves any hardware
352 	 * bits already set by default in "CLK_CTRL_REG" after reset.
353 	 */
354 	if (!trans->cfg->apmg_not_supported) {
355 		iwl_write_prph(trans, APMG_CLK_EN_REG,
356 			       APMG_CLK_VAL_DMA_CLK_RQT);
357 		udelay(20);
358 
359 		/* Disable L1-Active */
360 		iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
361 				  APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
362 
363 		/* Clear the interrupt in APMG if the NIC is in RFKILL */
364 		iwl_write_prph(trans, APMG_RTC_INT_STT_REG,
365 			       APMG_RTC_INT_STT_RFKILL);
366 	}
367 
368 	set_bit(STATUS_DEVICE_ENABLED, &trans->status);
369 
370 	return 0;
371 }
372 
373 /*
374  * Enable LP XTAL to avoid HW bug where device may consume much power if
375  * FW is not loaded after device reset. LP XTAL is disabled by default
376  * after device HW reset. Do it only if XTAL is fed by internal source.
377  * Configure device's "persistence" mode to avoid resetting XTAL again when
378  * SHRD_HW_RST occurs in S3.
379  */
380 static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
381 {
382 	int ret;
383 	u32 apmg_gp1_reg;
384 	u32 apmg_xtal_cfg_reg;
385 	u32 dl_cfg_reg;
386 
387 	/* Force XTAL ON */
388 	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
389 				 CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
390 
391 	ret = iwl_trans_pcie_sw_reset(trans, true);
392 
393 	if (!ret)
394 		ret = iwl_finish_nic_init(trans);
395 
396 	if (WARN_ON(ret)) {
397 		/* Release XTAL ON request */
398 		__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
399 					   CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
400 		return;
401 	}
402 
403 	/*
404 	 * Clear "disable persistence" to avoid LP XTAL resetting when
405 	 * SHRD_HW_RST is applied in S3.
406 	 */
407 	iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
408 				    APMG_PCIDEV_STT_VAL_PERSIST_DIS);
409 
410 	/*
411 	 * Force APMG XTAL to be active to prevent its disabling by HW
412 	 * caused by APMG idle state.
413 	 */
414 	apmg_xtal_cfg_reg = iwl_trans_pcie_read_shr(trans,
415 						    SHR_APMG_XTAL_CFG_REG);
416 	iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
417 				 apmg_xtal_cfg_reg |
418 				 SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
419 
420 	ret = iwl_trans_pcie_sw_reset(trans, true);
421 	if (ret)
422 		IWL_ERR(trans,
423 			"iwl_pcie_apm_lp_xtal_enable: failed to retake NIC ownership\n");
424 
425 	/* Enable LP XTAL by indirect access through CSR */
426 	apmg_gp1_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_GP1_REG);
427 	iwl_trans_pcie_write_shr(trans, SHR_APMG_GP1_REG, apmg_gp1_reg |
428 				 SHR_APMG_GP1_WF_XTAL_LP_EN |
429 				 SHR_APMG_GP1_CHICKEN_BIT_SELECT);
430 
431 	/* Clear delay line clock power up */
432 	dl_cfg_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_DL_CFG_REG);
433 	iwl_trans_pcie_write_shr(trans, SHR_APMG_DL_CFG_REG, dl_cfg_reg &
434 				 ~SHR_APMG_DL_CFG_DL_CLOCK_POWER_UP);
435 
436 	/*
437 	 * Enable persistence mode to avoid LP XTAL resetting when
438 	 * SHRD_HW_RST is applied in S3.
439 	 */
440 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
441 		    CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
442 
443 	/*
444 	 * Clear "initialization complete" bit to move adapter from
445 	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
446 	 */
447 	iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
448 
449 	/* Activates XTAL resources monitor */
450 	__iwl_trans_pcie_set_bit(trans, CSR_MONITOR_CFG_REG,
451 				 CSR_MONITOR_XTAL_RESOURCES);
452 
453 	/* Release XTAL ON request */
454 	__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
455 				   CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
456 	udelay(10);
457 
458 	/* Release APMG XTAL */
459 	iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
460 				 apmg_xtal_cfg_reg &
461 				 ~SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
462 }
463 
464 void iwl_pcie_apm_stop_master(struct iwl_trans *trans)
465 {
466 	int ret;
467 
468 	/* stop device's busmaster DMA activity */
469 
470 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
471 		iwl_set_bit(trans, CSR_GP_CNTRL,
472 			    CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_REQ);
473 
474 		ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
475 				   CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_STATUS,
476 				   CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_STATUS,
477 				   100);
478 		msleep(100);
479 	} else {
480 		iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_STOP_MASTER);
481 
482 		ret = iwl_poll_bit(trans, CSR_RESET,
483 				   CSR_RESET_REG_FLAG_MASTER_DISABLED,
484 				   CSR_RESET_REG_FLAG_MASTER_DISABLED, 100);
485 	}
486 
487 	if (ret < 0)
488 		IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n");
489 
490 	IWL_DEBUG_INFO(trans, "stop master\n");
491 }
492 
493 static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
494 {
495 	IWL_DEBUG_INFO(trans, "Stop card, put in low power state\n");
496 
497 	if (op_mode_leave) {
498 		if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
499 			iwl_pcie_apm_init(trans);
500 
501 		/* inform ME that we are leaving */
502 		if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000)
503 			iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
504 					  APMG_PCIDEV_STT_VAL_WAKE_ME);
505 		else if (trans->trans_cfg->device_family >=
506 			 IWL_DEVICE_FAMILY_8000) {
507 			iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
508 				    CSR_RESET_LINK_PWR_MGMT_DISABLED);
509 			iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
510 				    CSR_HW_IF_CONFIG_REG_PREPARE |
511 				    CSR_HW_IF_CONFIG_REG_ENABLE_PME);
512 			mdelay(1);
513 			iwl_clear_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
514 				      CSR_RESET_LINK_PWR_MGMT_DISABLED);
515 		}
516 		mdelay(5);
517 	}
518 
519 	clear_bit(STATUS_DEVICE_ENABLED, &trans->status);
520 
521 	/* Stop device's DMA activity */
522 	iwl_pcie_apm_stop_master(trans);
523 
524 	if (trans->cfg->lp_xtal_workaround) {
525 		iwl_pcie_apm_lp_xtal_enable(trans);
526 		return;
527 	}
528 
529 	iwl_trans_pcie_sw_reset(trans, false);
530 
531 	/*
532 	 * Clear "initialization complete" bit to move adapter from
533 	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
534 	 */
535 	iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
536 }
537 
538 static int iwl_pcie_nic_init(struct iwl_trans *trans)
539 {
540 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
541 	int ret;
542 
543 	/* nic_init */
544 	spin_lock_bh(&trans_pcie->irq_lock);
545 	ret = iwl_pcie_apm_init(trans);
546 	spin_unlock_bh(&trans_pcie->irq_lock);
547 
548 	if (ret)
549 		return ret;
550 
551 	iwl_pcie_set_pwr(trans, false);
552 
553 	iwl_op_mode_nic_config(trans->op_mode);
554 
555 	/* Allocate the RX queue, or reset if it is already allocated */
556 	ret = iwl_pcie_rx_init(trans);
557 	if (ret)
558 		return ret;
559 
560 	/* Allocate or reset and init all Tx and Command queues */
561 	if (iwl_pcie_tx_init(trans)) {
562 		iwl_pcie_rx_free(trans);
563 		return -ENOMEM;
564 	}
565 
566 	if (trans->trans_cfg->base_params->shadow_reg_enable) {
567 		/* enable shadow regs in HW */
568 		iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, 0x800FFFFF);
569 		IWL_DEBUG_INFO(trans, "Enabling shadow registers in device\n");
570 	}
571 
572 	return 0;
573 }
574 
575 #define HW_READY_TIMEOUT (50)
576 
577 /* Note: returns poll_bit return value, which is >= 0 if success */
578 static int iwl_pcie_set_hw_ready(struct iwl_trans *trans)
579 {
580 	int ret;
581 
582 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
583 		    CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);
584 
585 	/* See if we got it */
586 	ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
587 			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
588 			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
589 			   HW_READY_TIMEOUT);
590 
591 	if (ret >= 0)
592 		iwl_set_bit(trans, CSR_MBOX_SET_REG, CSR_MBOX_SET_REG_OS_ALIVE);
593 
594 	IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : "");
595 	return ret;
596 }
597 
598 /* Note: returns standard 0/-ERROR code */
599 int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
600 {
601 	int ret;
602 	int iter;
603 
604 	IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n");
605 
606 	ret = iwl_pcie_set_hw_ready(trans);
607 	/* If the card is ready, exit 0 */
608 	if (ret >= 0) {
609 		trans->csme_own = false;
610 		return 0;
611 	}
612 
613 	iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
614 		    CSR_RESET_LINK_PWR_MGMT_DISABLED);
615 	usleep_range(1000, 2000);
616 
617 	for (iter = 0; iter < 10; iter++) {
618 		int t = 0;
619 
620 		/* If HW is not ready, prepare the conditions to check again */
621 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
622 			    CSR_HW_IF_CONFIG_REG_PREPARE);
623 
624 		do {
625 			ret = iwl_pcie_set_hw_ready(trans);
626 			if (ret >= 0) {
627 				trans->csme_own = false;
628 				return 0;
629 			}
630 
631 			if (iwl_mei_is_connected()) {
632 				IWL_DEBUG_INFO(trans,
633 					       "Couldn't prepare the card but SAP is connected\n");
634 				trans->csme_own = true;
635 				if (trans->trans_cfg->device_family !=
636 				    IWL_DEVICE_FAMILY_9000)
637 					IWL_ERR(trans,
638 						"SAP not supported for this NIC family\n");
639 
640 				return -EBUSY;
641 			}
642 
643 			usleep_range(200, 1000);
644 			t += 200;
645 		} while (t < 150000);
646 		msleep(25);
647 	}
648 
649 	IWL_ERR(trans, "Couldn't prepare the card\n");
650 
651 	return ret;
652 }
653 
654 /*
655  * ucode
656  */
657 static void iwl_pcie_load_firmware_chunk_fh(struct iwl_trans *trans,
658 					    u32 dst_addr, dma_addr_t phy_addr,
659 					    u32 byte_cnt)
660 {
661 	iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
662 		    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
663 
664 	iwl_write32(trans, FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL),
665 		    dst_addr);
666 
667 	iwl_write32(trans, FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL),
668 		    phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
669 
670 	iwl_write32(trans, FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL),
671 		    (iwl_get_dma_hi_addr(phy_addr)
672 			<< FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
673 
674 	iwl_write32(trans, FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL),
675 		    BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM) |
676 		    BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX) |
677 		    FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
678 
679 	iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
680 		    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
681 		    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
682 		    FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
683 }
684 
685 static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans,
686 					u32 dst_addr, dma_addr_t phy_addr,
687 					u32 byte_cnt)
688 {
689 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
690 	int ret;
691 
692 	trans_pcie->ucode_write_complete = false;
693 
694 	if (!iwl_trans_grab_nic_access(trans))
695 		return -EIO;
696 
697 	iwl_pcie_load_firmware_chunk_fh(trans, dst_addr, phy_addr,
698 					byte_cnt);
699 	iwl_trans_release_nic_access(trans);
700 
701 	ret = wait_event_timeout(trans_pcie->ucode_write_waitq,
702 				 trans_pcie->ucode_write_complete, 5 * HZ);
703 	if (!ret) {
704 		IWL_ERR(trans, "Failed to load firmware chunk!\n");
705 		iwl_trans_pcie_dump_regs(trans);
706 		return -ETIMEDOUT;
707 	}
708 
709 	return 0;
710 }
711 
712 static int iwl_pcie_load_section(struct iwl_trans *trans, u8 section_num,
713 			    const struct fw_desc *section)
714 {
715 	u8 *v_addr;
716 	dma_addr_t p_addr;
717 	u32 offset, chunk_sz = min_t(u32, FH_MEM_TB_MAX_LENGTH, section->len);
718 	int ret = 0;
719 
720 	IWL_DEBUG_FW(trans, "[%d] uCode section being loaded...\n",
721 		     section_num);
722 
723 	v_addr = dma_alloc_coherent(trans->dev, chunk_sz, &p_addr,
724 				    GFP_KERNEL | __GFP_NOWARN);
725 	if (!v_addr) {
726 		IWL_DEBUG_INFO(trans, "Falling back to small chunks of DMA\n");
727 		chunk_sz = PAGE_SIZE;
728 		v_addr = dma_alloc_coherent(trans->dev, chunk_sz,
729 					    &p_addr, GFP_KERNEL);
730 		if (!v_addr)
731 			return -ENOMEM;
732 	}
733 
734 	for (offset = 0; offset < section->len; offset += chunk_sz) {
735 		u32 copy_size, dst_addr;
736 		bool extended_addr = false;
737 
738 		copy_size = min_t(u32, chunk_sz, section->len - offset);
739 		dst_addr = section->offset + offset;
740 
741 		if (dst_addr >= IWL_FW_MEM_EXTENDED_START &&
742 		    dst_addr <= IWL_FW_MEM_EXTENDED_END)
743 			extended_addr = true;
744 
745 		if (extended_addr)
746 			iwl_set_bits_prph(trans, LMPM_CHICK,
747 					  LMPM_CHICK_EXTENDED_ADDR_SPACE);
748 
749 		memcpy(v_addr, (const u8 *)section->data + offset, copy_size);
750 		ret = iwl_pcie_load_firmware_chunk(trans, dst_addr, p_addr,
751 						   copy_size);
752 
753 		if (extended_addr)
754 			iwl_clear_bits_prph(trans, LMPM_CHICK,
755 					    LMPM_CHICK_EXTENDED_ADDR_SPACE);
756 
757 		if (ret) {
758 			IWL_ERR(trans,
759 				"Could not load the [%d] uCode section\n",
760 				section_num);
761 			break;
762 		}
763 	}
764 
765 	dma_free_coherent(trans->dev, chunk_sz, v_addr, p_addr);
766 	return ret;
767 }
768 
769 static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans,
770 					   const struct fw_img *image,
771 					   int cpu,
772 					   int *first_ucode_section)
773 {
774 	int shift_param;
775 	int i, ret = 0, sec_num = 0x1;
776 	u32 val, last_read_idx = 0;
777 
778 	if (cpu == 1) {
779 		shift_param = 0;
780 		*first_ucode_section = 0;
781 	} else {
782 		shift_param = 16;
783 		(*first_ucode_section)++;
784 	}
785 
786 	for (i = *first_ucode_section; i < image->num_sec; i++) {
787 		last_read_idx = i;
788 
789 		/*
790 		 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
791 		 * CPU1 to CPU2.
792 		 * PAGING_SEPARATOR_SECTION delimiter - separate between
793 		 * CPU2 non paged to CPU2 paging sec.
794 		 */
795 		if (!image->sec[i].data ||
796 		    image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
797 		    image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
798 			IWL_DEBUG_FW(trans,
799 				     "Break since Data not valid or Empty section, sec = %d\n",
800 				     i);
801 			break;
802 		}
803 
804 		ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
805 		if (ret)
806 			return ret;
807 
808 		/* Notify ucode of loaded section number and status */
809 		val = iwl_read_direct32(trans, FH_UCODE_LOAD_STATUS);
810 		val = val | (sec_num << shift_param);
811 		iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, val);
812 
813 		sec_num = (sec_num << 1) | 0x1;
814 	}
815 
816 	*first_ucode_section = last_read_idx;
817 
818 	iwl_enable_interrupts(trans);
819 
820 	if (trans->trans_cfg->use_tfh) {
821 		if (cpu == 1)
822 			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
823 				       0xFFFF);
824 		else
825 			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
826 				       0xFFFFFFFF);
827 	} else {
828 		if (cpu == 1)
829 			iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
830 					   0xFFFF);
831 		else
832 			iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
833 					   0xFFFFFFFF);
834 	}
835 
836 	return 0;
837 }
838 
839 static int iwl_pcie_load_cpu_sections(struct iwl_trans *trans,
840 				      const struct fw_img *image,
841 				      int cpu,
842 				      int *first_ucode_section)
843 {
844 	int i, ret = 0;
845 	u32 last_read_idx = 0;
846 
847 	if (cpu == 1)
848 		*first_ucode_section = 0;
849 	else
850 		(*first_ucode_section)++;
851 
852 	for (i = *first_ucode_section; i < image->num_sec; i++) {
853 		last_read_idx = i;
854 
855 		/*
856 		 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
857 		 * CPU1 to CPU2.
858 		 * PAGING_SEPARATOR_SECTION delimiter - separate between
859 		 * CPU2 non paged to CPU2 paging sec.
860 		 */
861 		if (!image->sec[i].data ||
862 		    image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
863 		    image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
864 			IWL_DEBUG_FW(trans,
865 				     "Break since Data not valid or Empty section, sec = %d\n",
866 				     i);
867 			break;
868 		}
869 
870 		ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
871 		if (ret)
872 			return ret;
873 	}
874 
875 	*first_ucode_section = last_read_idx;
876 
877 	return 0;
878 }
879 
880 static void iwl_pcie_apply_destination_ini(struct iwl_trans *trans)
881 {
882 	enum iwl_fw_ini_allocation_id alloc_id = IWL_FW_INI_ALLOCATION_ID_DBGC1;
883 	struct iwl_fw_ini_allocation_tlv *fw_mon_cfg =
884 		&trans->dbg.fw_mon_cfg[alloc_id];
885 	struct iwl_dram_data *frag;
886 
887 	if (!iwl_trans_dbg_ini_valid(trans))
888 		return;
889 
890 	if (le32_to_cpu(fw_mon_cfg->buf_location) ==
891 	    IWL_FW_INI_LOCATION_SRAM_PATH) {
892 		IWL_DEBUG_FW(trans, "WRT: Applying SMEM buffer destination\n");
893 		/* set sram monitor by enabling bit 7 */
894 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
895 			    CSR_HW_IF_CONFIG_REG_BIT_MONITOR_SRAM);
896 
897 		return;
898 	}
899 
900 	if (le32_to_cpu(fw_mon_cfg->buf_location) !=
901 	    IWL_FW_INI_LOCATION_DRAM_PATH ||
902 	    !trans->dbg.fw_mon_ini[alloc_id].num_frags)
903 		return;
904 
905 	frag = &trans->dbg.fw_mon_ini[alloc_id].frags[0];
906 
907 	IWL_DEBUG_FW(trans, "WRT: Applying DRAM destination (alloc_id=%u)\n",
908 		     alloc_id);
909 
910 	iwl_write_umac_prph(trans, MON_BUFF_BASE_ADDR_VER2,
911 			    frag->physical >> MON_BUFF_SHIFT_VER2);
912 	iwl_write_umac_prph(trans, MON_BUFF_END_ADDR_VER2,
913 			    (frag->physical + frag->size - 256) >>
914 			    MON_BUFF_SHIFT_VER2);
915 }
916 
917 void iwl_pcie_apply_destination(struct iwl_trans *trans)
918 {
919 	const struct iwl_fw_dbg_dest_tlv_v1 *dest = trans->dbg.dest_tlv;
920 	const struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
921 	int i;
922 
923 	if (iwl_trans_dbg_ini_valid(trans)) {
924 		iwl_pcie_apply_destination_ini(trans);
925 		return;
926 	}
927 
928 	IWL_INFO(trans, "Applying debug destination %s\n",
929 		 get_fw_dbg_mode_string(dest->monitor_mode));
930 
931 	if (dest->monitor_mode == EXTERNAL_MODE)
932 		iwl_pcie_alloc_fw_monitor(trans, dest->size_power);
933 	else
934 		IWL_WARN(trans, "PCI should have external buffer debug\n");
935 
936 	for (i = 0; i < trans->dbg.n_dest_reg; i++) {
937 		u32 addr = le32_to_cpu(dest->reg_ops[i].addr);
938 		u32 val = le32_to_cpu(dest->reg_ops[i].val);
939 
940 		switch (dest->reg_ops[i].op) {
941 		case CSR_ASSIGN:
942 			iwl_write32(trans, addr, val);
943 			break;
944 		case CSR_SETBIT:
945 			iwl_set_bit(trans, addr, BIT(val));
946 			break;
947 		case CSR_CLEARBIT:
948 			iwl_clear_bit(trans, addr, BIT(val));
949 			break;
950 		case PRPH_ASSIGN:
951 			iwl_write_prph(trans, addr, val);
952 			break;
953 		case PRPH_SETBIT:
954 			iwl_set_bits_prph(trans, addr, BIT(val));
955 			break;
956 		case PRPH_CLEARBIT:
957 			iwl_clear_bits_prph(trans, addr, BIT(val));
958 			break;
959 		case PRPH_BLOCKBIT:
960 			if (iwl_read_prph(trans, addr) & BIT(val)) {
961 				IWL_ERR(trans,
962 					"BIT(%u) in address 0x%x is 1, stopping FW configuration\n",
963 					val, addr);
964 				goto monitor;
965 			}
966 			break;
967 		default:
968 			IWL_ERR(trans, "FW debug - unknown OP %d\n",
969 				dest->reg_ops[i].op);
970 			break;
971 		}
972 	}
973 
974 monitor:
975 	if (dest->monitor_mode == EXTERNAL_MODE && fw_mon->size) {
976 		iwl_write_prph(trans, le32_to_cpu(dest->base_reg),
977 			       fw_mon->physical >> dest->base_shift);
978 		if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
979 			iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
980 				       (fw_mon->physical + fw_mon->size -
981 					256) >> dest->end_shift);
982 		else
983 			iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
984 				       (fw_mon->physical + fw_mon->size) >>
985 				       dest->end_shift);
986 	}
987 }
988 
989 static int iwl_pcie_load_given_ucode(struct iwl_trans *trans,
990 				const struct fw_img *image)
991 {
992 	int ret = 0;
993 	int first_ucode_section;
994 
995 	IWL_DEBUG_FW(trans, "working with %s CPU\n",
996 		     image->is_dual_cpus ? "Dual" : "Single");
997 
998 	/* load to FW the binary non secured sections of CPU1 */
999 	ret = iwl_pcie_load_cpu_sections(trans, image, 1, &first_ucode_section);
1000 	if (ret)
1001 		return ret;
1002 
1003 	if (image->is_dual_cpus) {
1004 		/* set CPU2 header address */
1005 		iwl_write_prph(trans,
1006 			       LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
1007 			       LMPM_SECURE_CPU2_HDR_MEM_SPACE);
1008 
1009 		/* load to FW the binary sections of CPU2 */
1010 		ret = iwl_pcie_load_cpu_sections(trans, image, 2,
1011 						 &first_ucode_section);
1012 		if (ret)
1013 			return ret;
1014 	}
1015 
1016 	if (iwl_pcie_dbg_on(trans))
1017 		iwl_pcie_apply_destination(trans);
1018 
1019 	iwl_enable_interrupts(trans);
1020 
1021 	/* release CPU reset */
1022 	iwl_write32(trans, CSR_RESET, 0);
1023 
1024 	return 0;
1025 }
1026 
1027 static int iwl_pcie_load_given_ucode_8000(struct iwl_trans *trans,
1028 					  const struct fw_img *image)
1029 {
1030 	int ret = 0;
1031 	int first_ucode_section;
1032 
1033 	IWL_DEBUG_FW(trans, "working with %s CPU\n",
1034 		     image->is_dual_cpus ? "Dual" : "Single");
1035 
1036 	if (iwl_pcie_dbg_on(trans))
1037 		iwl_pcie_apply_destination(trans);
1038 
1039 	IWL_DEBUG_POWER(trans, "Original WFPM value = 0x%08X\n",
1040 			iwl_read_prph(trans, WFPM_GP2));
1041 
1042 	/*
1043 	 * Set default value. On resume reading the values that were
1044 	 * zeored can provide debug data on the resume flow.
1045 	 * This is for debugging only and has no functional impact.
1046 	 */
1047 	iwl_write_prph(trans, WFPM_GP2, 0x01010101);
1048 
1049 	/* configure the ucode to be ready to get the secured image */
1050 	/* release CPU reset */
1051 	iwl_write_prph(trans, RELEASE_CPU_RESET, RELEASE_CPU_RESET_BIT);
1052 
1053 	/* load to FW the binary Secured sections of CPU1 */
1054 	ret = iwl_pcie_load_cpu_sections_8000(trans, image, 1,
1055 					      &first_ucode_section);
1056 	if (ret)
1057 		return ret;
1058 
1059 	/* load to FW the binary sections of CPU2 */
1060 	return iwl_pcie_load_cpu_sections_8000(trans, image, 2,
1061 					       &first_ucode_section);
1062 }
1063 
1064 bool iwl_pcie_check_hw_rf_kill(struct iwl_trans *trans)
1065 {
1066 	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1067 	bool hw_rfkill = iwl_is_rfkill_set(trans);
1068 	bool prev = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1069 	bool report;
1070 
1071 	if (hw_rfkill) {
1072 		set_bit(STATUS_RFKILL_HW, &trans->status);
1073 		set_bit(STATUS_RFKILL_OPMODE, &trans->status);
1074 	} else {
1075 		clear_bit(STATUS_RFKILL_HW, &trans->status);
1076 		if (trans_pcie->opmode_down)
1077 			clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
1078 	}
1079 
1080 	report = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1081 
1082 	if (prev != report)
1083 		iwl_trans_pcie_rf_kill(trans, report);
1084 
1085 	return hw_rfkill;
1086 }
1087 
1088 struct iwl_causes_list {
1089 	u16 mask_reg;
1090 	u8 bit;
1091 	u8 addr;
1092 };
1093 
1094 #define IWL_CAUSE(reg, mask)						\
1095 	{								\
1096 		.mask_reg = reg,					\
1097 		.bit = ilog2(mask),					\
1098 		.addr = ilog2(mask) +					\
1099 			((reg) == CSR_MSIX_FH_INT_MASK_AD ? -16 :	\
1100 			 (reg) == CSR_MSIX_HW_INT_MASK_AD ? 16 :	\
1101 			 0xffff),	/* causes overflow warning */	\
1102 	}
1103 
1104 static const struct iwl_causes_list causes_list_common[] = {
1105 	IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_D2S_CH0_NUM),
1106 	IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_D2S_CH1_NUM),
1107 	IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_S2D),
1108 	IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_FH_ERR),
1109 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_ALIVE),
1110 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_WAKEUP),
1111 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_RESET_DONE),
1112 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_CT_KILL),
1113 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_RF_KILL),
1114 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_PERIODIC),
1115 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_SCD),
1116 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_FH_TX),
1117 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_HW_ERR),
1118 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_HAP),
1119 };
1120 
1121 static const struct iwl_causes_list causes_list_pre_bz[] = {
1122 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_SW_ERR),
1123 };
1124 
1125 static const struct iwl_causes_list causes_list_bz[] = {
1126 	IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ),
1127 };
1128 
1129 static void iwl_pcie_map_list(struct iwl_trans *trans,
1130 			      const struct iwl_causes_list *causes,
1131 			      int arr_size, int val)
1132 {
1133 	int i;
1134 
1135 	for (i = 0; i < arr_size; i++) {
1136 		iwl_write8(trans, CSR_MSIX_IVAR(causes[i].addr), val);
1137 		iwl_clear_bit(trans, causes[i].mask_reg,
1138 			      BIT(causes[i].bit));
1139 	}
1140 }
1141 
1142 static void iwl_pcie_map_non_rx_causes(struct iwl_trans *trans)
1143 {
1144 	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1145 	int val = trans_pcie->def_irq | MSIX_NON_AUTO_CLEAR_CAUSE;
1146 	/*
1147 	 * Access all non RX causes and map them to the default irq.
1148 	 * In case we are missing at least one interrupt vector,
1149 	 * the first interrupt vector will serve non-RX and FBQ causes.
1150 	 */
1151 	iwl_pcie_map_list(trans, causes_list_common,
1152 			  ARRAY_SIZE(causes_list_common), val);
1153 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
1154 		iwl_pcie_map_list(trans, causes_list_bz,
1155 				  ARRAY_SIZE(causes_list_bz), val);
1156 	else
1157 		iwl_pcie_map_list(trans, causes_list_pre_bz,
1158 				  ARRAY_SIZE(causes_list_pre_bz), val);
1159 }
1160 
1161 static void iwl_pcie_map_rx_causes(struct iwl_trans *trans)
1162 {
1163 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1164 	u32 offset =
1165 		trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 1 : 0;
1166 	u32 val, idx;
1167 
1168 	/*
1169 	 * The first RX queue - fallback queue, which is designated for
1170 	 * management frame, command responses etc, is always mapped to the
1171 	 * first interrupt vector. The other RX queues are mapped to
1172 	 * the other (N - 2) interrupt vectors.
1173 	 */
1174 	val = BIT(MSIX_FH_INT_CAUSES_Q(0));
1175 	for (idx = 1; idx < trans->num_rx_queues; idx++) {
1176 		iwl_write8(trans, CSR_MSIX_RX_IVAR(idx),
1177 			   MSIX_FH_INT_CAUSES_Q(idx - offset));
1178 		val |= BIT(MSIX_FH_INT_CAUSES_Q(idx));
1179 	}
1180 	iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~val);
1181 
1182 	val = MSIX_FH_INT_CAUSES_Q(0);
1183 	if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX)
1184 		val |= MSIX_NON_AUTO_CLEAR_CAUSE;
1185 	iwl_write8(trans, CSR_MSIX_RX_IVAR(0), val);
1186 
1187 	if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS)
1188 		iwl_write8(trans, CSR_MSIX_RX_IVAR(1), val);
1189 }
1190 
1191 void iwl_pcie_conf_msix_hw(struct iwl_trans_pcie *trans_pcie)
1192 {
1193 	struct iwl_trans *trans = trans_pcie->trans;
1194 
1195 	if (!trans_pcie->msix_enabled) {
1196 		if (trans->trans_cfg->mq_rx_supported &&
1197 		    test_bit(STATUS_DEVICE_ENABLED, &trans->status))
1198 			iwl_write_umac_prph(trans, UREG_CHICK,
1199 					    UREG_CHICK_MSI_ENABLE);
1200 		return;
1201 	}
1202 	/*
1203 	 * The IVAR table needs to be configured again after reset,
1204 	 * but if the device is disabled, we can't write to
1205 	 * prph.
1206 	 */
1207 	if (test_bit(STATUS_DEVICE_ENABLED, &trans->status))
1208 		iwl_write_umac_prph(trans, UREG_CHICK, UREG_CHICK_MSIX_ENABLE);
1209 
1210 	/*
1211 	 * Each cause from the causes list above and the RX causes is
1212 	 * represented as a byte in the IVAR table. The first nibble
1213 	 * represents the bound interrupt vector of the cause, the second
1214 	 * represents no auto clear for this cause. This will be set if its
1215 	 * interrupt vector is bound to serve other causes.
1216 	 */
1217 	iwl_pcie_map_rx_causes(trans);
1218 
1219 	iwl_pcie_map_non_rx_causes(trans);
1220 }
1221 
1222 static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie)
1223 {
1224 	struct iwl_trans *trans = trans_pcie->trans;
1225 
1226 	iwl_pcie_conf_msix_hw(trans_pcie);
1227 
1228 	if (!trans_pcie->msix_enabled)
1229 		return;
1230 
1231 	trans_pcie->fh_init_mask = ~iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD);
1232 	trans_pcie->fh_mask = trans_pcie->fh_init_mask;
1233 	trans_pcie->hw_init_mask = ~iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD);
1234 	trans_pcie->hw_mask = trans_pcie->hw_init_mask;
1235 }
1236 
1237 static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans)
1238 {
1239 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1240 
1241 	lockdep_assert_held(&trans_pcie->mutex);
1242 
1243 	if (trans_pcie->is_down)
1244 		return;
1245 
1246 	trans_pcie->is_down = true;
1247 
1248 	/* tell the device to stop sending interrupts */
1249 	iwl_disable_interrupts(trans);
1250 
1251 	/* device going down, Stop using ICT table */
1252 	iwl_pcie_disable_ict(trans);
1253 
1254 	/*
1255 	 * If a HW restart happens during firmware loading,
1256 	 * then the firmware loading might call this function
1257 	 * and later it might be called again due to the
1258 	 * restart. So don't process again if the device is
1259 	 * already dead.
1260 	 */
1261 	if (test_and_clear_bit(STATUS_DEVICE_ENABLED, &trans->status)) {
1262 		IWL_DEBUG_INFO(trans,
1263 			       "DEVICE_ENABLED bit was set and is now cleared\n");
1264 		iwl_pcie_rx_napi_sync(trans);
1265 		iwl_pcie_tx_stop(trans);
1266 		iwl_pcie_rx_stop(trans);
1267 
1268 		/* Power-down device's busmaster DMA clocks */
1269 		if (!trans->cfg->apmg_not_supported) {
1270 			iwl_write_prph(trans, APMG_CLK_DIS_REG,
1271 				       APMG_CLK_VAL_DMA_CLK_RQT);
1272 			udelay(5);
1273 		}
1274 	}
1275 
1276 	/* Make sure (redundant) we've released our request to stay awake */
1277 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
1278 		iwl_clear_bit(trans, CSR_GP_CNTRL,
1279 			      CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
1280 	else
1281 		iwl_clear_bit(trans, CSR_GP_CNTRL,
1282 			      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1283 
1284 	/* Stop the device, and put it in low power state */
1285 	iwl_pcie_apm_stop(trans, false);
1286 
1287 	/* re-take ownership to prevent other users from stealing the device */
1288 	iwl_trans_pcie_sw_reset(trans, true);
1289 
1290 	/*
1291 	 * Upon stop, the IVAR table gets erased, so msi-x won't
1292 	 * work. This causes a bug in RF-KILL flows, since the interrupt
1293 	 * that enables radio won't fire on the correct irq, and the
1294 	 * driver won't be able to handle the interrupt.
1295 	 * Configure the IVAR table again after reset.
1296 	 */
1297 	iwl_pcie_conf_msix_hw(trans_pcie);
1298 
1299 	/*
1300 	 * Upon stop, the APM issues an interrupt if HW RF kill is set.
1301 	 * This is a bug in certain verions of the hardware.
1302 	 * Certain devices also keep sending HW RF kill interrupt all
1303 	 * the time, unless the interrupt is ACKed even if the interrupt
1304 	 * should be masked. Re-ACK all the interrupts here.
1305 	 */
1306 	iwl_disable_interrupts(trans);
1307 
1308 	/* clear all status bits */
1309 	clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1310 	clear_bit(STATUS_INT_ENABLED, &trans->status);
1311 	clear_bit(STATUS_TPOWER_PMI, &trans->status);
1312 
1313 	/*
1314 	 * Even if we stop the HW, we still want the RF kill
1315 	 * interrupt
1316 	 */
1317 	iwl_enable_rfkill_int(trans);
1318 }
1319 
1320 void iwl_pcie_synchronize_irqs(struct iwl_trans *trans)
1321 {
1322 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1323 
1324 	if (trans_pcie->msix_enabled) {
1325 		int i;
1326 
1327 		for (i = 0; i < trans_pcie->alloc_vecs; i++)
1328 			synchronize_irq(trans_pcie->msix_entries[i].vector);
1329 	} else {
1330 		synchronize_irq(trans_pcie->pci_dev->irq);
1331 	}
1332 }
1333 
1334 static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
1335 				   const struct fw_img *fw, bool run_in_rfkill)
1336 {
1337 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1338 	bool hw_rfkill;
1339 	int ret;
1340 
1341 	/* This may fail if AMT took ownership of the device */
1342 	if (iwl_pcie_prepare_card_hw(trans)) {
1343 		IWL_WARN(trans, "Exit HW not ready\n");
1344 		return -EIO;
1345 	}
1346 
1347 	iwl_enable_rfkill_int(trans);
1348 
1349 	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1350 
1351 	/*
1352 	 * We enabled the RF-Kill interrupt and the handler may very
1353 	 * well be running. Disable the interrupts to make sure no other
1354 	 * interrupt can be fired.
1355 	 */
1356 	iwl_disable_interrupts(trans);
1357 
1358 	/* Make sure it finished running */
1359 	iwl_pcie_synchronize_irqs(trans);
1360 
1361 	mutex_lock(&trans_pcie->mutex);
1362 
1363 	/* If platform's RF_KILL switch is NOT set to KILL */
1364 	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
1365 	if (hw_rfkill && !run_in_rfkill) {
1366 		ret = -ERFKILL;
1367 		goto out;
1368 	}
1369 
1370 	/* Someone called stop_device, don't try to start_fw */
1371 	if (trans_pcie->is_down) {
1372 		IWL_WARN(trans,
1373 			 "Can't start_fw since the HW hasn't been started\n");
1374 		ret = -EIO;
1375 		goto out;
1376 	}
1377 
1378 	/* make sure rfkill handshake bits are cleared */
1379 	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1380 	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR,
1381 		    CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
1382 
1383 	/* clear (again), then enable host interrupts */
1384 	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1385 
1386 	ret = iwl_pcie_nic_init(trans);
1387 	if (ret) {
1388 		IWL_ERR(trans, "Unable to init nic\n");
1389 		goto out;
1390 	}
1391 
1392 	/*
1393 	 * Now, we load the firmware and don't want to be interrupted, even
1394 	 * by the RF-Kill interrupt (hence mask all the interrupt besides the
1395 	 * FH_TX interrupt which is needed to load the firmware). If the
1396 	 * RF-Kill switch is toggled, we will find out after having loaded
1397 	 * the firmware and return the proper value to the caller.
1398 	 */
1399 	iwl_enable_fw_load_int(trans);
1400 
1401 	/* really make sure rfkill handshake bits are cleared */
1402 	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1403 	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1404 
1405 	/* Load the given image to the HW */
1406 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
1407 		ret = iwl_pcie_load_given_ucode_8000(trans, fw);
1408 	else
1409 		ret = iwl_pcie_load_given_ucode(trans, fw);
1410 
1411 	/* re-check RF-Kill state since we may have missed the interrupt */
1412 	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
1413 	if (hw_rfkill && !run_in_rfkill)
1414 		ret = -ERFKILL;
1415 
1416 out:
1417 	mutex_unlock(&trans_pcie->mutex);
1418 	return ret;
1419 }
1420 
1421 static void iwl_trans_pcie_fw_alive(struct iwl_trans *trans, u32 scd_addr)
1422 {
1423 	iwl_pcie_reset_ict(trans);
1424 	iwl_pcie_tx_start(trans, scd_addr);
1425 }
1426 
1427 void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans,
1428 				       bool was_in_rfkill)
1429 {
1430 	bool hw_rfkill;
1431 
1432 	/*
1433 	 * Check again since the RF kill state may have changed while
1434 	 * all the interrupts were disabled, in this case we couldn't
1435 	 * receive the RF kill interrupt and update the state in the
1436 	 * op_mode.
1437 	 * Don't call the op_mode if the rkfill state hasn't changed.
1438 	 * This allows the op_mode to call stop_device from the rfkill
1439 	 * notification without endless recursion. Under very rare
1440 	 * circumstances, we might have a small recursion if the rfkill
1441 	 * state changed exactly now while we were called from stop_device.
1442 	 * This is very unlikely but can happen and is supported.
1443 	 */
1444 	hw_rfkill = iwl_is_rfkill_set(trans);
1445 	if (hw_rfkill) {
1446 		set_bit(STATUS_RFKILL_HW, &trans->status);
1447 		set_bit(STATUS_RFKILL_OPMODE, &trans->status);
1448 	} else {
1449 		clear_bit(STATUS_RFKILL_HW, &trans->status);
1450 		clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
1451 	}
1452 	if (hw_rfkill != was_in_rfkill)
1453 		iwl_trans_pcie_rf_kill(trans, hw_rfkill);
1454 }
1455 
1456 static void iwl_trans_pcie_stop_device(struct iwl_trans *trans)
1457 {
1458 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1459 	bool was_in_rfkill;
1460 
1461 	iwl_op_mode_time_point(trans->op_mode,
1462 			       IWL_FW_INI_TIME_POINT_HOST_DEVICE_DISABLE,
1463 			       NULL);
1464 
1465 	mutex_lock(&trans_pcie->mutex);
1466 	trans_pcie->opmode_down = true;
1467 	was_in_rfkill = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1468 	_iwl_trans_pcie_stop_device(trans);
1469 	iwl_trans_pcie_handle_stop_rfkill(trans, was_in_rfkill);
1470 	mutex_unlock(&trans_pcie->mutex);
1471 }
1472 
1473 void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state)
1474 {
1475 	struct iwl_trans_pcie __maybe_unused *trans_pcie =
1476 		IWL_TRANS_GET_PCIE_TRANS(trans);
1477 
1478 	lockdep_assert_held(&trans_pcie->mutex);
1479 
1480 	IWL_WARN(trans, "reporting RF_KILL (radio %s)\n",
1481 		 state ? "disabled" : "enabled");
1482 	if (iwl_op_mode_hw_rf_kill(trans->op_mode, state)) {
1483 		if (trans->trans_cfg->gen2)
1484 			_iwl_trans_pcie_gen2_stop_device(trans);
1485 		else
1486 			_iwl_trans_pcie_stop_device(trans);
1487 	}
1488 }
1489 
1490 void iwl_pcie_d3_complete_suspend(struct iwl_trans *trans,
1491 				  bool test, bool reset)
1492 {
1493 	iwl_disable_interrupts(trans);
1494 
1495 	/*
1496 	 * in testing mode, the host stays awake and the
1497 	 * hardware won't be reset (not even partially)
1498 	 */
1499 	if (test)
1500 		return;
1501 
1502 	iwl_pcie_disable_ict(trans);
1503 
1504 	iwl_pcie_synchronize_irqs(trans);
1505 
1506 	iwl_clear_bit(trans, CSR_GP_CNTRL,
1507 		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1508 	iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
1509 
1510 	if (reset) {
1511 		/*
1512 		 * reset TX queues -- some of their registers reset during S3
1513 		 * so if we don't reset everything here the D3 image would try
1514 		 * to execute some invalid memory upon resume
1515 		 */
1516 		iwl_trans_pcie_tx_reset(trans);
1517 	}
1518 
1519 	iwl_pcie_set_pwr(trans, true);
1520 }
1521 
1522 static int iwl_pcie_d3_handshake(struct iwl_trans *trans, bool suspend)
1523 {
1524 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1525 	int ret;
1526 
1527 	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210)
1528 		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
1529 				    suspend ? UREG_DOORBELL_TO_ISR6_SUSPEND :
1530 					      UREG_DOORBELL_TO_ISR6_RESUME);
1531 	else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
1532 		iwl_write32(trans, CSR_IPC_SLEEP_CONTROL,
1533 			    suspend ? CSR_IPC_SLEEP_CONTROL_SUSPEND :
1534 				      CSR_IPC_SLEEP_CONTROL_RESUME);
1535 	else
1536 		return 0;
1537 
1538 	ret = wait_event_timeout(trans_pcie->sx_waitq,
1539 				 trans_pcie->sx_complete, 2 * HZ);
1540 
1541 	/* Invalidate it toward next suspend or resume */
1542 	trans_pcie->sx_complete = false;
1543 
1544 	if (!ret) {
1545 		IWL_ERR(trans, "Timeout %s D3\n",
1546 			suspend ? "entering" : "exiting");
1547 		return -ETIMEDOUT;
1548 	}
1549 
1550 	return 0;
1551 }
1552 
1553 static int iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test,
1554 				     bool reset)
1555 {
1556 	int ret;
1557 
1558 	if (!reset)
1559 		/* Enable persistence mode to avoid reset */
1560 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
1561 			    CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
1562 
1563 	ret = iwl_pcie_d3_handshake(trans, true);
1564 	if (ret)
1565 		return ret;
1566 
1567 	iwl_pcie_d3_complete_suspend(trans, test, reset);
1568 
1569 	return 0;
1570 }
1571 
1572 static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
1573 				    enum iwl_d3_status *status,
1574 				    bool test,  bool reset)
1575 {
1576 	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1577 	u32 val;
1578 	int ret;
1579 
1580 	if (test) {
1581 		iwl_enable_interrupts(trans);
1582 		*status = IWL_D3_STATUS_ALIVE;
1583 		ret = 0;
1584 		goto out;
1585 	}
1586 
1587 	iwl_set_bit(trans, CSR_GP_CNTRL,
1588 		    CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1589 
1590 	ret = iwl_finish_nic_init(trans);
1591 	if (ret)
1592 		return ret;
1593 
1594 	/*
1595 	 * Reconfigure IVAR table in case of MSIX or reset ict table in
1596 	 * MSI mode since HW reset erased it.
1597 	 * Also enables interrupts - none will happen as
1598 	 * the device doesn't know we're waking it up, only when
1599 	 * the opmode actually tells it after this call.
1600 	 */
1601 	iwl_pcie_conf_msix_hw(trans_pcie);
1602 	if (!trans_pcie->msix_enabled)
1603 		iwl_pcie_reset_ict(trans);
1604 	iwl_enable_interrupts(trans);
1605 
1606 	iwl_pcie_set_pwr(trans, false);
1607 
1608 	if (!reset) {
1609 		iwl_clear_bit(trans, CSR_GP_CNTRL,
1610 			      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1611 	} else {
1612 		iwl_trans_pcie_tx_reset(trans);
1613 
1614 		ret = iwl_pcie_rx_init(trans);
1615 		if (ret) {
1616 			IWL_ERR(trans,
1617 				"Failed to resume the device (RX reset)\n");
1618 			return ret;
1619 		}
1620 	}
1621 
1622 	IWL_DEBUG_POWER(trans, "WFPM value upon resume = 0x%08X\n",
1623 			iwl_read_umac_prph(trans, WFPM_GP2));
1624 
1625 	val = iwl_read32(trans, CSR_RESET);
1626 	if (val & CSR_RESET_REG_FLAG_NEVO_RESET)
1627 		*status = IWL_D3_STATUS_RESET;
1628 	else
1629 		*status = IWL_D3_STATUS_ALIVE;
1630 
1631 out:
1632 	if (*status == IWL_D3_STATUS_ALIVE)
1633 		ret = iwl_pcie_d3_handshake(trans, false);
1634 
1635 	return ret;
1636 }
1637 
1638 static void
1639 iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
1640 			    struct iwl_trans *trans,
1641 			    const struct iwl_cfg_trans_params *cfg_trans)
1642 {
1643 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1644 	int max_irqs, num_irqs, i, ret;
1645 	u16 pci_cmd;
1646 	u32 max_rx_queues = IWL_MAX_RX_HW_QUEUES;
1647 
1648 	if (!cfg_trans->mq_rx_supported)
1649 		goto enable_msi;
1650 
1651 	if (cfg_trans->device_family <= IWL_DEVICE_FAMILY_9000)
1652 		max_rx_queues = IWL_9000_MAX_RX_HW_QUEUES;
1653 
1654 	max_irqs = min_t(u32, num_online_cpus() + 2, max_rx_queues);
1655 	for (i = 0; i < max_irqs; i++)
1656 		trans_pcie->msix_entries[i].entry = i;
1657 
1658 	num_irqs = pci_enable_msix_range(pdev, trans_pcie->msix_entries,
1659 					 MSIX_MIN_INTERRUPT_VECTORS,
1660 					 max_irqs);
1661 	if (num_irqs < 0) {
1662 		IWL_DEBUG_INFO(trans,
1663 			       "Failed to enable msi-x mode (ret %d). Moving to msi mode.\n",
1664 			       num_irqs);
1665 		goto enable_msi;
1666 	}
1667 	trans_pcie->def_irq = (num_irqs == max_irqs) ? num_irqs - 1 : 0;
1668 
1669 	IWL_DEBUG_INFO(trans,
1670 		       "MSI-X enabled. %d interrupt vectors were allocated\n",
1671 		       num_irqs);
1672 
1673 	/*
1674 	 * In case the OS provides fewer interrupts than requested, different
1675 	 * causes will share the same interrupt vector as follows:
1676 	 * One interrupt less: non rx causes shared with FBQ.
1677 	 * Two interrupts less: non rx causes shared with FBQ and RSS.
1678 	 * More than two interrupts: we will use fewer RSS queues.
1679 	 */
1680 	if (num_irqs <= max_irqs - 2) {
1681 		trans_pcie->trans->num_rx_queues = num_irqs + 1;
1682 		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX |
1683 			IWL_SHARED_IRQ_FIRST_RSS;
1684 	} else if (num_irqs == max_irqs - 1) {
1685 		trans_pcie->trans->num_rx_queues = num_irqs;
1686 		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX;
1687 	} else {
1688 		trans_pcie->trans->num_rx_queues = num_irqs - 1;
1689 	}
1690 
1691 	IWL_DEBUG_INFO(trans,
1692 		       "MSI-X enabled with rx queues %d, vec mask 0x%x\n",
1693 		       trans_pcie->trans->num_rx_queues, trans_pcie->shared_vec_mask);
1694 
1695 	WARN_ON(trans_pcie->trans->num_rx_queues > IWL_MAX_RX_HW_QUEUES);
1696 
1697 	trans_pcie->alloc_vecs = num_irqs;
1698 	trans_pcie->msix_enabled = true;
1699 	return;
1700 
1701 enable_msi:
1702 	ret = pci_enable_msi(pdev);
1703 	if (ret) {
1704 		dev_err(&pdev->dev, "pci_enable_msi failed - %d\n", ret);
1705 		/* enable rfkill interrupt: hw bug w/a */
1706 		pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
1707 		if (pci_cmd & PCI_COMMAND_INTX_DISABLE) {
1708 			pci_cmd &= ~PCI_COMMAND_INTX_DISABLE;
1709 			pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
1710 		}
1711 	}
1712 }
1713 
1714 static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans)
1715 {
1716 	int iter_rx_q, i, ret, cpu, offset;
1717 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1718 
1719 	i = trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 0 : 1;
1720 	iter_rx_q = trans_pcie->trans->num_rx_queues - 1 + i;
1721 	offset = 1 + i;
1722 	for (; i < iter_rx_q ; i++) {
1723 		/*
1724 		 * Get the cpu prior to the place to search
1725 		 * (i.e. return will be > i - 1).
1726 		 */
1727 		cpu = cpumask_next(i - offset, cpu_online_mask);
1728 		cpumask_set_cpu(cpu, &trans_pcie->affinity_mask[i]);
1729 		ret = irq_set_affinity_hint(trans_pcie->msix_entries[i].vector,
1730 					    &trans_pcie->affinity_mask[i]);
1731 		if (ret)
1732 			IWL_ERR(trans_pcie->trans,
1733 				"Failed to set affinity mask for IRQ %d\n",
1734 				trans_pcie->msix_entries[i].vector);
1735 	}
1736 }
1737 
1738 static int iwl_pcie_init_msix_handler(struct pci_dev *pdev,
1739 				      struct iwl_trans_pcie *trans_pcie)
1740 {
1741 	int i;
1742 
1743 	for (i = 0; i < trans_pcie->alloc_vecs; i++) {
1744 		int ret;
1745 		struct msix_entry *msix_entry;
1746 		const char *qname = queue_name(&pdev->dev, trans_pcie, i);
1747 
1748 		if (!qname)
1749 			return -ENOMEM;
1750 
1751 		msix_entry = &trans_pcie->msix_entries[i];
1752 		ret = devm_request_threaded_irq(&pdev->dev,
1753 						msix_entry->vector,
1754 						iwl_pcie_msix_isr,
1755 						(i == trans_pcie->def_irq) ?
1756 						iwl_pcie_irq_msix_handler :
1757 						iwl_pcie_irq_rx_msix_handler,
1758 						IRQF_SHARED,
1759 						qname,
1760 						msix_entry);
1761 		if (ret) {
1762 			IWL_ERR(trans_pcie->trans,
1763 				"Error allocating IRQ %d\n", i);
1764 
1765 			return ret;
1766 		}
1767 	}
1768 	iwl_pcie_irq_set_affinity(trans_pcie->trans);
1769 
1770 	return 0;
1771 }
1772 
1773 static int iwl_trans_pcie_clear_persistence_bit(struct iwl_trans *trans)
1774 {
1775 	u32 hpm, wprot;
1776 
1777 	switch (trans->trans_cfg->device_family) {
1778 	case IWL_DEVICE_FAMILY_9000:
1779 		wprot = PREG_PRPH_WPROT_9000;
1780 		break;
1781 	case IWL_DEVICE_FAMILY_22000:
1782 		wprot = PREG_PRPH_WPROT_22000;
1783 		break;
1784 	default:
1785 		return 0;
1786 	}
1787 
1788 	hpm = iwl_read_umac_prph_no_grab(trans, HPM_DEBUG);
1789 	if (hpm != 0xa5a5a5a0 && (hpm & PERSISTENCE_BIT)) {
1790 		u32 wprot_val = iwl_read_umac_prph_no_grab(trans, wprot);
1791 
1792 		if (wprot_val & PREG_WFPM_ACCESS) {
1793 			IWL_ERR(trans,
1794 				"Error, can not clear persistence bit\n");
1795 			return -EPERM;
1796 		}
1797 		iwl_write_umac_prph_no_grab(trans, HPM_DEBUG,
1798 					    hpm & ~PERSISTENCE_BIT);
1799 	}
1800 
1801 	return 0;
1802 }
1803 
1804 static int iwl_pcie_gen2_force_power_gating(struct iwl_trans *trans)
1805 {
1806 	int ret;
1807 
1808 	ret = iwl_finish_nic_init(trans);
1809 	if (ret < 0)
1810 		return ret;
1811 
1812 	iwl_set_bits_prph(trans, HPM_HIPM_GEN_CFG,
1813 			  HPM_HIPM_GEN_CFG_CR_FORCE_ACTIVE);
1814 	udelay(20);
1815 	iwl_set_bits_prph(trans, HPM_HIPM_GEN_CFG,
1816 			  HPM_HIPM_GEN_CFG_CR_PG_EN |
1817 			  HPM_HIPM_GEN_CFG_CR_SLP_EN);
1818 	udelay(20);
1819 	iwl_clear_bits_prph(trans, HPM_HIPM_GEN_CFG,
1820 			    HPM_HIPM_GEN_CFG_CR_FORCE_ACTIVE);
1821 
1822 	return iwl_trans_pcie_sw_reset(trans, true);
1823 }
1824 
1825 static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans)
1826 {
1827 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1828 	int err;
1829 
1830 	lockdep_assert_held(&trans_pcie->mutex);
1831 
1832 	err = iwl_pcie_prepare_card_hw(trans);
1833 	if (err) {
1834 		IWL_ERR(trans, "Error while preparing HW: %d\n", err);
1835 		return err;
1836 	}
1837 
1838 	err = iwl_trans_pcie_clear_persistence_bit(trans);
1839 	if (err)
1840 		return err;
1841 
1842 	err = iwl_trans_pcie_sw_reset(trans, true);
1843 	if (err)
1844 		return err;
1845 
1846 	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000 &&
1847 	    trans->trans_cfg->integrated) {
1848 		err = iwl_pcie_gen2_force_power_gating(trans);
1849 		if (err)
1850 			return err;
1851 	}
1852 
1853 	err = iwl_pcie_apm_init(trans);
1854 	if (err)
1855 		return err;
1856 
1857 	iwl_pcie_init_msix(trans_pcie);
1858 
1859 	/* From now on, the op_mode will be kept updated about RF kill state */
1860 	iwl_enable_rfkill_int(trans);
1861 
1862 	trans_pcie->opmode_down = false;
1863 
1864 	/* Set is_down to false here so that...*/
1865 	trans_pcie->is_down = false;
1866 
1867 	/* ...rfkill can call stop_device and set it false if needed */
1868 	iwl_pcie_check_hw_rf_kill(trans);
1869 
1870 	return 0;
1871 }
1872 
1873 static int iwl_trans_pcie_start_hw(struct iwl_trans *trans)
1874 {
1875 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1876 	int ret;
1877 
1878 	mutex_lock(&trans_pcie->mutex);
1879 	ret = _iwl_trans_pcie_start_hw(trans);
1880 	mutex_unlock(&trans_pcie->mutex);
1881 
1882 	return ret;
1883 }
1884 
1885 static void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans)
1886 {
1887 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1888 
1889 	mutex_lock(&trans_pcie->mutex);
1890 
1891 	/* disable interrupts - don't enable HW RF kill interrupt */
1892 	iwl_disable_interrupts(trans);
1893 
1894 	iwl_pcie_apm_stop(trans, true);
1895 
1896 	iwl_disable_interrupts(trans);
1897 
1898 	iwl_pcie_disable_ict(trans);
1899 
1900 	mutex_unlock(&trans_pcie->mutex);
1901 
1902 	iwl_pcie_synchronize_irqs(trans);
1903 }
1904 
1905 static void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val)
1906 {
1907 	writeb(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1908 }
1909 
1910 static void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val)
1911 {
1912 	writel(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1913 }
1914 
1915 static u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs)
1916 {
1917 	return readl(IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1918 }
1919 
1920 static u32 iwl_trans_pcie_prph_msk(struct iwl_trans *trans)
1921 {
1922 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
1923 		return 0x00FFFFFF;
1924 	else
1925 		return 0x000FFFFF;
1926 }
1927 
1928 static u32 iwl_trans_pcie_read_prph(struct iwl_trans *trans, u32 reg)
1929 {
1930 	u32 mask = iwl_trans_pcie_prph_msk(trans);
1931 
1932 	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_RADDR,
1933 			       ((reg & mask) | (3 << 24)));
1934 	return iwl_trans_pcie_read32(trans, HBUS_TARG_PRPH_RDAT);
1935 }
1936 
1937 static void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr,
1938 				      u32 val)
1939 {
1940 	u32 mask = iwl_trans_pcie_prph_msk(trans);
1941 
1942 	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WADDR,
1943 			       ((addr & mask) | (3 << 24)));
1944 	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WDAT, val);
1945 }
1946 
1947 static void iwl_trans_pcie_configure(struct iwl_trans *trans,
1948 				     const struct iwl_trans_config *trans_cfg)
1949 {
1950 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1951 
1952 	/* free all first - we might be reconfigured for a different size */
1953 	iwl_pcie_free_rbs_pool(trans);
1954 
1955 	trans->txqs.cmd.q_id = trans_cfg->cmd_queue;
1956 	trans->txqs.cmd.fifo = trans_cfg->cmd_fifo;
1957 	trans->txqs.cmd.wdg_timeout = trans_cfg->cmd_q_wdg_timeout;
1958 	trans->txqs.page_offs = trans_cfg->cb_data_offs;
1959 	trans->txqs.dev_cmd_offs = trans_cfg->cb_data_offs + sizeof(void *);
1960 	trans->txqs.queue_alloc_cmd_ver = trans_cfg->queue_alloc_cmd_ver;
1961 
1962 	if (WARN_ON(trans_cfg->n_no_reclaim_cmds > MAX_NO_RECLAIM_CMDS))
1963 		trans_pcie->n_no_reclaim_cmds = 0;
1964 	else
1965 		trans_pcie->n_no_reclaim_cmds = trans_cfg->n_no_reclaim_cmds;
1966 	if (trans_pcie->n_no_reclaim_cmds)
1967 		memcpy(trans_pcie->no_reclaim_cmds, trans_cfg->no_reclaim_cmds,
1968 		       trans_pcie->n_no_reclaim_cmds * sizeof(u8));
1969 
1970 	trans_pcie->rx_buf_size = trans_cfg->rx_buf_size;
1971 	trans_pcie->rx_page_order =
1972 		iwl_trans_get_rb_size_order(trans_pcie->rx_buf_size);
1973 	trans_pcie->rx_buf_bytes =
1974 		iwl_trans_get_rb_size(trans_pcie->rx_buf_size);
1975 	trans_pcie->supported_dma_mask = DMA_BIT_MASK(12);
1976 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
1977 		trans_pcie->supported_dma_mask = DMA_BIT_MASK(11);
1978 
1979 	trans->txqs.bc_table_dword = trans_cfg->bc_table_dword;
1980 	trans_pcie->scd_set_active = trans_cfg->scd_set_active;
1981 
1982 	trans->command_groups = trans_cfg->command_groups;
1983 	trans->command_groups_size = trans_cfg->command_groups_size;
1984 
1985 	/* Initialize NAPI here - it should be before registering to mac80211
1986 	 * in the opmode but after the HW struct is allocated.
1987 	 * As this function may be called again in some corner cases don't
1988 	 * do anything if NAPI was already initialized.
1989 	 */
1990 	if (trans_pcie->napi_dev.reg_state != NETREG_DUMMY)
1991 		init_dummy_netdev(&trans_pcie->napi_dev);
1992 
1993 	trans_pcie->fw_reset_handshake = trans_cfg->fw_reset_handshake;
1994 }
1995 
1996 void iwl_trans_pcie_free(struct iwl_trans *trans)
1997 {
1998 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1999 	int i;
2000 
2001 	iwl_pcie_synchronize_irqs(trans);
2002 
2003 	if (trans->trans_cfg->gen2)
2004 		iwl_txq_gen2_tx_free(trans);
2005 	else
2006 		iwl_pcie_tx_free(trans);
2007 	iwl_pcie_rx_free(trans);
2008 
2009 	if (trans_pcie->rba.alloc_wq) {
2010 		destroy_workqueue(trans_pcie->rba.alloc_wq);
2011 		trans_pcie->rba.alloc_wq = NULL;
2012 	}
2013 
2014 	if (trans_pcie->msix_enabled) {
2015 		for (i = 0; i < trans_pcie->alloc_vecs; i++) {
2016 			irq_set_affinity_hint(
2017 				trans_pcie->msix_entries[i].vector,
2018 				NULL);
2019 		}
2020 
2021 		trans_pcie->msix_enabled = false;
2022 	} else {
2023 		iwl_pcie_free_ict(trans);
2024 	}
2025 
2026 	iwl_pcie_free_fw_monitor(trans);
2027 
2028 	if (trans_pcie->pnvm_dram.size)
2029 		dma_free_coherent(trans->dev, trans_pcie->pnvm_dram.size,
2030 				  trans_pcie->pnvm_dram.block,
2031 				  trans_pcie->pnvm_dram.physical);
2032 
2033 	if (trans_pcie->reduce_power_dram.size)
2034 		dma_free_coherent(trans->dev,
2035 				  trans_pcie->reduce_power_dram.size,
2036 				  trans_pcie->reduce_power_dram.block,
2037 				  trans_pcie->reduce_power_dram.physical);
2038 
2039 	mutex_destroy(&trans_pcie->mutex);
2040 	iwl_trans_free(trans);
2041 }
2042 
2043 static void iwl_trans_pcie_set_pmi(struct iwl_trans *trans, bool state)
2044 {
2045 	if (state)
2046 		set_bit(STATUS_TPOWER_PMI, &trans->status);
2047 	else
2048 		clear_bit(STATUS_TPOWER_PMI, &trans->status);
2049 }
2050 
2051 struct iwl_trans_pcie_removal {
2052 	struct pci_dev *pdev;
2053 	struct work_struct work;
2054 	bool rescan;
2055 };
2056 
2057 static void iwl_trans_pcie_removal_wk(struct work_struct *wk)
2058 {
2059 	struct iwl_trans_pcie_removal *removal =
2060 		container_of(wk, struct iwl_trans_pcie_removal, work);
2061 	struct pci_dev *pdev = removal->pdev;
2062 	static char *prop[] = {"EVENT=INACCESSIBLE", NULL};
2063 	struct pci_bus *bus = pdev->bus;
2064 
2065 	dev_err(&pdev->dev, "Device gone - attempting removal\n");
2066 	kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, prop);
2067 	pci_lock_rescan_remove();
2068 	pci_dev_put(pdev);
2069 	pci_stop_and_remove_bus_device(pdev);
2070 	if (removal->rescan)
2071 		pci_rescan_bus(bus->parent);
2072 	pci_unlock_rescan_remove();
2073 
2074 	kfree(removal);
2075 	module_put(THIS_MODULE);
2076 }
2077 
2078 void iwl_trans_pcie_remove(struct iwl_trans *trans, bool rescan)
2079 {
2080 	struct iwl_trans_pcie_removal *removal;
2081 
2082 	if (test_bit(STATUS_TRANS_DEAD, &trans->status))
2083 		return;
2084 
2085 	IWL_ERR(trans, "Device gone - scheduling removal!\n");
2086 
2087 	/*
2088 	 * get a module reference to avoid doing this
2089 	 * while unloading anyway and to avoid
2090 	 * scheduling a work with code that's being
2091 	 * removed.
2092 	 */
2093 	if (!try_module_get(THIS_MODULE)) {
2094 		IWL_ERR(trans,
2095 			"Module is being unloaded - abort\n");
2096 		return;
2097 	}
2098 
2099 	removal = kzalloc(sizeof(*removal), GFP_ATOMIC);
2100 	if (!removal) {
2101 		module_put(THIS_MODULE);
2102 		return;
2103 	}
2104 	/*
2105 	 * we don't need to clear this flag, because
2106 	 * the trans will be freed and reallocated.
2107 	 */
2108 	set_bit(STATUS_TRANS_DEAD, &trans->status);
2109 
2110 	removal->pdev = to_pci_dev(trans->dev);
2111 	removal->rescan = rescan;
2112 	INIT_WORK(&removal->work, iwl_trans_pcie_removal_wk);
2113 	pci_dev_get(removal->pdev);
2114 	schedule_work(&removal->work);
2115 }
2116 EXPORT_SYMBOL(iwl_trans_pcie_remove);
2117 
2118 /*
2119  * This version doesn't disable BHs but rather assumes they're
2120  * already disabled.
2121  */
2122 bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
2123 {
2124 	int ret;
2125 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2126 	u32 write = CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ;
2127 	u32 mask = CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
2128 		   CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP;
2129 	u32 poll = CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN;
2130 
2131 	spin_lock(&trans_pcie->reg_lock);
2132 
2133 	if (trans_pcie->cmd_hold_nic_awake)
2134 		goto out;
2135 
2136 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
2137 		write = CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ;
2138 		mask = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS;
2139 		poll = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS;
2140 	}
2141 
2142 	/* this bit wakes up the NIC */
2143 	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL, write);
2144 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
2145 		udelay(2);
2146 
2147 	/*
2148 	 * These bits say the device is running, and should keep running for
2149 	 * at least a short while (at least as long as MAC_ACCESS_REQ stays 1),
2150 	 * but they do not indicate that embedded SRAM is restored yet;
2151 	 * HW with volatile SRAM must save/restore contents to/from
2152 	 * host DRAM when sleeping/waking for power-saving.
2153 	 * Each direction takes approximately 1/4 millisecond; with this
2154 	 * overhead, it's a good idea to grab and hold MAC_ACCESS_REQUEST if a
2155 	 * series of register accesses are expected (e.g. reading Event Log),
2156 	 * to keep device from sleeping.
2157 	 *
2158 	 * CSR_UCODE_DRV_GP1 register bit MAC_SLEEP == 0 indicates that
2159 	 * SRAM is okay/restored.  We don't check that here because this call
2160 	 * is just for hardware register access; but GP1 MAC_SLEEP
2161 	 * check is a good idea before accessing the SRAM of HW with
2162 	 * volatile SRAM (e.g. reading Event Log).
2163 	 *
2164 	 * 5000 series and later (including 1000 series) have non-volatile SRAM,
2165 	 * and do not save/restore SRAM when power cycling.
2166 	 */
2167 	ret = iwl_poll_bit(trans, CSR_GP_CNTRL, poll, mask, 15000);
2168 	if (unlikely(ret < 0)) {
2169 		u32 cntrl = iwl_read32(trans, CSR_GP_CNTRL);
2170 
2171 		WARN_ONCE(1,
2172 			  "Timeout waiting for hardware access (CSR_GP_CNTRL 0x%08x)\n",
2173 			  cntrl);
2174 
2175 		iwl_trans_pcie_dump_regs(trans);
2176 
2177 		if (iwlwifi_mod_params.remove_when_gone && cntrl == ~0U)
2178 			iwl_trans_pcie_remove(trans, false);
2179 		else
2180 			iwl_write32(trans, CSR_RESET,
2181 				    CSR_RESET_REG_FLAG_FORCE_NMI);
2182 
2183 		spin_unlock(&trans_pcie->reg_lock);
2184 		return false;
2185 	}
2186 
2187 out:
2188 	/*
2189 	 * Fool sparse by faking we release the lock - sparse will
2190 	 * track nic_access anyway.
2191 	 */
2192 	__release(&trans_pcie->reg_lock);
2193 	return true;
2194 }
2195 
2196 static bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
2197 {
2198 	bool ret;
2199 
2200 	local_bh_disable();
2201 	ret = __iwl_trans_pcie_grab_nic_access(trans);
2202 	if (ret) {
2203 		/* keep BHs disabled until iwl_trans_pcie_release_nic_access */
2204 		return ret;
2205 	}
2206 	local_bh_enable();
2207 	return false;
2208 }
2209 
2210 static void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
2211 {
2212 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2213 
2214 	lockdep_assert_held(&trans_pcie->reg_lock);
2215 
2216 	/*
2217 	 * Fool sparse by faking we acquiring the lock - sparse will
2218 	 * track nic_access anyway.
2219 	 */
2220 	__acquire(&trans_pcie->reg_lock);
2221 
2222 	if (trans_pcie->cmd_hold_nic_awake)
2223 		goto out;
2224 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
2225 		__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
2226 					   CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
2227 	else
2228 		__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
2229 					   CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
2230 	/*
2231 	 * Above we read the CSR_GP_CNTRL register, which will flush
2232 	 * any previous writes, but we need the write that clears the
2233 	 * MAC_ACCESS_REQ bit to be performed before any other writes
2234 	 * scheduled on different CPUs (after we drop reg_lock).
2235 	 */
2236 out:
2237 	spin_unlock_bh(&trans_pcie->reg_lock);
2238 }
2239 
2240 static int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
2241 				   void *buf, int dwords)
2242 {
2243 	int offs = 0;
2244 	u32 *vals = buf;
2245 
2246 	while (offs < dwords) {
2247 		/* limit the time we spin here under lock to 1/2s */
2248 		unsigned long end = jiffies + HZ / 2;
2249 		bool resched = false;
2250 
2251 		if (iwl_trans_grab_nic_access(trans)) {
2252 			iwl_write32(trans, HBUS_TARG_MEM_RADDR,
2253 				    addr + 4 * offs);
2254 
2255 			while (offs < dwords) {
2256 				vals[offs] = iwl_read32(trans,
2257 							HBUS_TARG_MEM_RDAT);
2258 				offs++;
2259 
2260 				if (time_after(jiffies, end)) {
2261 					resched = true;
2262 					break;
2263 				}
2264 			}
2265 			iwl_trans_release_nic_access(trans);
2266 
2267 			if (resched)
2268 				cond_resched();
2269 		} else {
2270 			return -EBUSY;
2271 		}
2272 	}
2273 
2274 	return 0;
2275 }
2276 
2277 static int iwl_trans_pcie_write_mem(struct iwl_trans *trans, u32 addr,
2278 				    const void *buf, int dwords)
2279 {
2280 	int offs, ret = 0;
2281 	const u32 *vals = buf;
2282 
2283 	if (iwl_trans_grab_nic_access(trans)) {
2284 		iwl_write32(trans, HBUS_TARG_MEM_WADDR, addr);
2285 		for (offs = 0; offs < dwords; offs++)
2286 			iwl_write32(trans, HBUS_TARG_MEM_WDAT,
2287 				    vals ? vals[offs] : 0);
2288 		iwl_trans_release_nic_access(trans);
2289 	} else {
2290 		ret = -EBUSY;
2291 	}
2292 	return ret;
2293 }
2294 
2295 static int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs,
2296 					u32 *val)
2297 {
2298 	return pci_read_config_dword(IWL_TRANS_GET_PCIE_TRANS(trans)->pci_dev,
2299 				     ofs, val);
2300 }
2301 
2302 static void iwl_trans_pcie_block_txq_ptrs(struct iwl_trans *trans, bool block)
2303 {
2304 	int i;
2305 
2306 	for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) {
2307 		struct iwl_txq *txq = trans->txqs.txq[i];
2308 
2309 		if (i == trans->txqs.cmd.q_id)
2310 			continue;
2311 
2312 		spin_lock_bh(&txq->lock);
2313 
2314 		if (!block && !(WARN_ON_ONCE(!txq->block))) {
2315 			txq->block--;
2316 			if (!txq->block) {
2317 				iwl_write32(trans, HBUS_TARG_WRPTR,
2318 					    txq->write_ptr | (i << 8));
2319 			}
2320 		} else if (block) {
2321 			txq->block++;
2322 		}
2323 
2324 		spin_unlock_bh(&txq->lock);
2325 	}
2326 }
2327 
2328 #define IWL_FLUSH_WAIT_MS	2000
2329 
2330 static int iwl_trans_pcie_rxq_dma_data(struct iwl_trans *trans, int queue,
2331 				       struct iwl_trans_rxq_dma_data *data)
2332 {
2333 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2334 
2335 	if (queue >= trans->num_rx_queues || !trans_pcie->rxq)
2336 		return -EINVAL;
2337 
2338 	data->fr_bd_cb = trans_pcie->rxq[queue].bd_dma;
2339 	data->urbd_stts_wrptr = trans_pcie->rxq[queue].rb_stts_dma;
2340 	data->ur_bd_cb = trans_pcie->rxq[queue].used_bd_dma;
2341 	data->fr_bd_wid = 0;
2342 
2343 	return 0;
2344 }
2345 
2346 static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, int txq_idx)
2347 {
2348 	struct iwl_txq *txq;
2349 	unsigned long now = jiffies;
2350 	bool overflow_tx;
2351 	u8 wr_ptr;
2352 
2353 	/* Make sure the NIC is still alive in the bus */
2354 	if (test_bit(STATUS_TRANS_DEAD, &trans->status))
2355 		return -ENODEV;
2356 
2357 	if (!test_bit(txq_idx, trans->txqs.queue_used))
2358 		return -EINVAL;
2359 
2360 	IWL_DEBUG_TX_QUEUES(trans, "Emptying queue %d...\n", txq_idx);
2361 	txq = trans->txqs.txq[txq_idx];
2362 
2363 	spin_lock_bh(&txq->lock);
2364 	overflow_tx = txq->overflow_tx ||
2365 		      !skb_queue_empty(&txq->overflow_q);
2366 	spin_unlock_bh(&txq->lock);
2367 
2368 	wr_ptr = READ_ONCE(txq->write_ptr);
2369 
2370 	while ((txq->read_ptr != READ_ONCE(txq->write_ptr) ||
2371 		overflow_tx) &&
2372 	       !time_after(jiffies,
2373 			   now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS))) {
2374 		u8 write_ptr = READ_ONCE(txq->write_ptr);
2375 
2376 		/*
2377 		 * If write pointer moved during the wait, warn only
2378 		 * if the TX came from op mode. In case TX came from
2379 		 * trans layer (overflow TX) don't warn.
2380 		 */
2381 		if (WARN_ONCE(wr_ptr != write_ptr && !overflow_tx,
2382 			      "WR pointer moved while flushing %d -> %d\n",
2383 			      wr_ptr, write_ptr))
2384 			return -ETIMEDOUT;
2385 		wr_ptr = write_ptr;
2386 
2387 		usleep_range(1000, 2000);
2388 
2389 		spin_lock_bh(&txq->lock);
2390 		overflow_tx = txq->overflow_tx ||
2391 			      !skb_queue_empty(&txq->overflow_q);
2392 		spin_unlock_bh(&txq->lock);
2393 	}
2394 
2395 	if (txq->read_ptr != txq->write_ptr) {
2396 		IWL_ERR(trans,
2397 			"fail to flush all tx fifo queues Q %d\n", txq_idx);
2398 		iwl_txq_log_scd_error(trans, txq);
2399 		return -ETIMEDOUT;
2400 	}
2401 
2402 	IWL_DEBUG_TX_QUEUES(trans, "Queue %d is now empty.\n", txq_idx);
2403 
2404 	return 0;
2405 }
2406 
2407 static int iwl_trans_pcie_wait_txqs_empty(struct iwl_trans *trans, u32 txq_bm)
2408 {
2409 	int cnt;
2410 	int ret = 0;
2411 
2412 	/* waiting for all the tx frames complete might take a while */
2413 	for (cnt = 0;
2414 	     cnt < trans->trans_cfg->base_params->num_of_queues;
2415 	     cnt++) {
2416 
2417 		if (cnt == trans->txqs.cmd.q_id)
2418 			continue;
2419 		if (!test_bit(cnt, trans->txqs.queue_used))
2420 			continue;
2421 		if (!(BIT(cnt) & txq_bm))
2422 			continue;
2423 
2424 		ret = iwl_trans_pcie_wait_txq_empty(trans, cnt);
2425 		if (ret)
2426 			break;
2427 	}
2428 
2429 	return ret;
2430 }
2431 
2432 static void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg,
2433 					 u32 mask, u32 value)
2434 {
2435 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2436 
2437 	spin_lock_bh(&trans_pcie->reg_lock);
2438 	__iwl_trans_pcie_set_bits_mask(trans, reg, mask, value);
2439 	spin_unlock_bh(&trans_pcie->reg_lock);
2440 }
2441 
2442 static const char *get_csr_string(int cmd)
2443 {
2444 #define IWL_CMD(x) case x: return #x
2445 	switch (cmd) {
2446 	IWL_CMD(CSR_HW_IF_CONFIG_REG);
2447 	IWL_CMD(CSR_INT_COALESCING);
2448 	IWL_CMD(CSR_INT);
2449 	IWL_CMD(CSR_INT_MASK);
2450 	IWL_CMD(CSR_FH_INT_STATUS);
2451 	IWL_CMD(CSR_GPIO_IN);
2452 	IWL_CMD(CSR_RESET);
2453 	IWL_CMD(CSR_GP_CNTRL);
2454 	IWL_CMD(CSR_HW_REV);
2455 	IWL_CMD(CSR_EEPROM_REG);
2456 	IWL_CMD(CSR_EEPROM_GP);
2457 	IWL_CMD(CSR_OTP_GP_REG);
2458 	IWL_CMD(CSR_GIO_REG);
2459 	IWL_CMD(CSR_GP_UCODE_REG);
2460 	IWL_CMD(CSR_GP_DRIVER_REG);
2461 	IWL_CMD(CSR_UCODE_DRV_GP1);
2462 	IWL_CMD(CSR_UCODE_DRV_GP2);
2463 	IWL_CMD(CSR_LED_REG);
2464 	IWL_CMD(CSR_DRAM_INT_TBL_REG);
2465 	IWL_CMD(CSR_GIO_CHICKEN_BITS);
2466 	IWL_CMD(CSR_ANA_PLL_CFG);
2467 	IWL_CMD(CSR_HW_REV_WA_REG);
2468 	IWL_CMD(CSR_MONITOR_STATUS_REG);
2469 	IWL_CMD(CSR_DBG_HPET_MEM_REG);
2470 	default:
2471 		return "UNKNOWN";
2472 	}
2473 #undef IWL_CMD
2474 }
2475 
2476 void iwl_pcie_dump_csr(struct iwl_trans *trans)
2477 {
2478 	int i;
2479 	static const u32 csr_tbl[] = {
2480 		CSR_HW_IF_CONFIG_REG,
2481 		CSR_INT_COALESCING,
2482 		CSR_INT,
2483 		CSR_INT_MASK,
2484 		CSR_FH_INT_STATUS,
2485 		CSR_GPIO_IN,
2486 		CSR_RESET,
2487 		CSR_GP_CNTRL,
2488 		CSR_HW_REV,
2489 		CSR_EEPROM_REG,
2490 		CSR_EEPROM_GP,
2491 		CSR_OTP_GP_REG,
2492 		CSR_GIO_REG,
2493 		CSR_GP_UCODE_REG,
2494 		CSR_GP_DRIVER_REG,
2495 		CSR_UCODE_DRV_GP1,
2496 		CSR_UCODE_DRV_GP2,
2497 		CSR_LED_REG,
2498 		CSR_DRAM_INT_TBL_REG,
2499 		CSR_GIO_CHICKEN_BITS,
2500 		CSR_ANA_PLL_CFG,
2501 		CSR_MONITOR_STATUS_REG,
2502 		CSR_HW_REV_WA_REG,
2503 		CSR_DBG_HPET_MEM_REG
2504 	};
2505 	IWL_ERR(trans, "CSR values:\n");
2506 	IWL_ERR(trans, "(2nd byte of CSR_INT_COALESCING is "
2507 		"CSR_INT_PERIODIC_REG)\n");
2508 	for (i = 0; i <  ARRAY_SIZE(csr_tbl); i++) {
2509 		IWL_ERR(trans, "  %25s: 0X%08x\n",
2510 			get_csr_string(csr_tbl[i]),
2511 			iwl_read32(trans, csr_tbl[i]));
2512 	}
2513 }
2514 
2515 #ifdef CONFIG_IWLWIFI_DEBUGFS
2516 /* create and remove of files */
2517 #define DEBUGFS_ADD_FILE(name, parent, mode) do {			\
2518 	debugfs_create_file(#name, mode, parent, trans,			\
2519 			    &iwl_dbgfs_##name##_ops);			\
2520 } while (0)
2521 
2522 /* file operation */
2523 #define DEBUGFS_READ_FILE_OPS(name)					\
2524 static const struct file_operations iwl_dbgfs_##name##_ops = {		\
2525 	.read = iwl_dbgfs_##name##_read,				\
2526 	.open = simple_open,						\
2527 	.llseek = generic_file_llseek,					\
2528 };
2529 
2530 #define DEBUGFS_WRITE_FILE_OPS(name)                                    \
2531 static const struct file_operations iwl_dbgfs_##name##_ops = {          \
2532 	.write = iwl_dbgfs_##name##_write,                              \
2533 	.open = simple_open,						\
2534 	.llseek = generic_file_llseek,					\
2535 };
2536 
2537 #define DEBUGFS_READ_WRITE_FILE_OPS(name)				\
2538 static const struct file_operations iwl_dbgfs_##name##_ops = {		\
2539 	.write = iwl_dbgfs_##name##_write,				\
2540 	.read = iwl_dbgfs_##name##_read,				\
2541 	.open = simple_open,						\
2542 	.llseek = generic_file_llseek,					\
2543 };
2544 
2545 struct iwl_dbgfs_tx_queue_priv {
2546 	struct iwl_trans *trans;
2547 };
2548 
2549 struct iwl_dbgfs_tx_queue_state {
2550 	loff_t pos;
2551 };
2552 
2553 static void *iwl_dbgfs_tx_queue_seq_start(struct seq_file *seq, loff_t *pos)
2554 {
2555 	struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
2556 	struct iwl_dbgfs_tx_queue_state *state;
2557 
2558 	if (*pos >= priv->trans->trans_cfg->base_params->num_of_queues)
2559 		return NULL;
2560 
2561 	state = kmalloc(sizeof(*state), GFP_KERNEL);
2562 	if (!state)
2563 		return NULL;
2564 	state->pos = *pos;
2565 	return state;
2566 }
2567 
2568 static void *iwl_dbgfs_tx_queue_seq_next(struct seq_file *seq,
2569 					 void *v, loff_t *pos)
2570 {
2571 	struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
2572 	struct iwl_dbgfs_tx_queue_state *state = v;
2573 
2574 	*pos = ++state->pos;
2575 
2576 	if (*pos >= priv->trans->trans_cfg->base_params->num_of_queues)
2577 		return NULL;
2578 
2579 	return state;
2580 }
2581 
2582 static void iwl_dbgfs_tx_queue_seq_stop(struct seq_file *seq, void *v)
2583 {
2584 	kfree(v);
2585 }
2586 
2587 static int iwl_dbgfs_tx_queue_seq_show(struct seq_file *seq, void *v)
2588 {
2589 	struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
2590 	struct iwl_dbgfs_tx_queue_state *state = v;
2591 	struct iwl_trans *trans = priv->trans;
2592 	struct iwl_txq *txq = trans->txqs.txq[state->pos];
2593 
2594 	seq_printf(seq, "hwq %.3u: used=%d stopped=%d ",
2595 		   (unsigned int)state->pos,
2596 		   !!test_bit(state->pos, trans->txqs.queue_used),
2597 		   !!test_bit(state->pos, trans->txqs.queue_stopped));
2598 	if (txq)
2599 		seq_printf(seq,
2600 			   "read=%u write=%u need_update=%d frozen=%d n_window=%d ampdu=%d",
2601 			   txq->read_ptr, txq->write_ptr,
2602 			   txq->need_update, txq->frozen,
2603 			   txq->n_window, txq->ampdu);
2604 	else
2605 		seq_puts(seq, "(unallocated)");
2606 
2607 	if (state->pos == trans->txqs.cmd.q_id)
2608 		seq_puts(seq, " (HCMD)");
2609 	seq_puts(seq, "\n");
2610 
2611 	return 0;
2612 }
2613 
2614 static const struct seq_operations iwl_dbgfs_tx_queue_seq_ops = {
2615 	.start = iwl_dbgfs_tx_queue_seq_start,
2616 	.next = iwl_dbgfs_tx_queue_seq_next,
2617 	.stop = iwl_dbgfs_tx_queue_seq_stop,
2618 	.show = iwl_dbgfs_tx_queue_seq_show,
2619 };
2620 
2621 static int iwl_dbgfs_tx_queue_open(struct inode *inode, struct file *filp)
2622 {
2623 	struct iwl_dbgfs_tx_queue_priv *priv;
2624 
2625 	priv = __seq_open_private(filp, &iwl_dbgfs_tx_queue_seq_ops,
2626 				  sizeof(*priv));
2627 
2628 	if (!priv)
2629 		return -ENOMEM;
2630 
2631 	priv->trans = inode->i_private;
2632 	return 0;
2633 }
2634 
2635 static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
2636 				       char __user *user_buf,
2637 				       size_t count, loff_t *ppos)
2638 {
2639 	struct iwl_trans *trans = file->private_data;
2640 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2641 	char *buf;
2642 	int pos = 0, i, ret;
2643 	size_t bufsz;
2644 
2645 	bufsz = sizeof(char) * 121 * trans->num_rx_queues;
2646 
2647 	if (!trans_pcie->rxq)
2648 		return -EAGAIN;
2649 
2650 	buf = kzalloc(bufsz, GFP_KERNEL);
2651 	if (!buf)
2652 		return -ENOMEM;
2653 
2654 	for (i = 0; i < trans->num_rx_queues && pos < bufsz; i++) {
2655 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
2656 
2657 		pos += scnprintf(buf + pos, bufsz - pos, "queue#: %2d\n",
2658 				 i);
2659 		pos += scnprintf(buf + pos, bufsz - pos, "\tread: %u\n",
2660 				 rxq->read);
2661 		pos += scnprintf(buf + pos, bufsz - pos, "\twrite: %u\n",
2662 				 rxq->write);
2663 		pos += scnprintf(buf + pos, bufsz - pos, "\twrite_actual: %u\n",
2664 				 rxq->write_actual);
2665 		pos += scnprintf(buf + pos, bufsz - pos, "\tneed_update: %2d\n",
2666 				 rxq->need_update);
2667 		pos += scnprintf(buf + pos, bufsz - pos, "\tfree_count: %u\n",
2668 				 rxq->free_count);
2669 		if (rxq->rb_stts) {
2670 			u32 r =	__le16_to_cpu(iwl_get_closed_rb_stts(trans,
2671 								     rxq));
2672 			pos += scnprintf(buf + pos, bufsz - pos,
2673 					 "\tclosed_rb_num: %u\n",
2674 					 r & 0x0FFF);
2675 		} else {
2676 			pos += scnprintf(buf + pos, bufsz - pos,
2677 					 "\tclosed_rb_num: Not Allocated\n");
2678 		}
2679 	}
2680 	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2681 	kfree(buf);
2682 
2683 	return ret;
2684 }
2685 
2686 static ssize_t iwl_dbgfs_interrupt_read(struct file *file,
2687 					char __user *user_buf,
2688 					size_t count, loff_t *ppos)
2689 {
2690 	struct iwl_trans *trans = file->private_data;
2691 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2692 	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
2693 
2694 	int pos = 0;
2695 	char *buf;
2696 	int bufsz = 24 * 64; /* 24 items * 64 char per item */
2697 	ssize_t ret;
2698 
2699 	buf = kzalloc(bufsz, GFP_KERNEL);
2700 	if (!buf)
2701 		return -ENOMEM;
2702 
2703 	pos += scnprintf(buf + pos, bufsz - pos,
2704 			"Interrupt Statistics Report:\n");
2705 
2706 	pos += scnprintf(buf + pos, bufsz - pos, "HW Error:\t\t\t %u\n",
2707 		isr_stats->hw);
2708 	pos += scnprintf(buf + pos, bufsz - pos, "SW Error:\t\t\t %u\n",
2709 		isr_stats->sw);
2710 	if (isr_stats->sw || isr_stats->hw) {
2711 		pos += scnprintf(buf + pos, bufsz - pos,
2712 			"\tLast Restarting Code:  0x%X\n",
2713 			isr_stats->err_code);
2714 	}
2715 #ifdef CONFIG_IWLWIFI_DEBUG
2716 	pos += scnprintf(buf + pos, bufsz - pos, "Frame transmitted:\t\t %u\n",
2717 		isr_stats->sch);
2718 	pos += scnprintf(buf + pos, bufsz - pos, "Alive interrupt:\t\t %u\n",
2719 		isr_stats->alive);
2720 #endif
2721 	pos += scnprintf(buf + pos, bufsz - pos,
2722 		"HW RF KILL switch toggled:\t %u\n", isr_stats->rfkill);
2723 
2724 	pos += scnprintf(buf + pos, bufsz - pos, "CT KILL:\t\t\t %u\n",
2725 		isr_stats->ctkill);
2726 
2727 	pos += scnprintf(buf + pos, bufsz - pos, "Wakeup Interrupt:\t\t %u\n",
2728 		isr_stats->wakeup);
2729 
2730 	pos += scnprintf(buf + pos, bufsz - pos,
2731 		"Rx command responses:\t\t %u\n", isr_stats->rx);
2732 
2733 	pos += scnprintf(buf + pos, bufsz - pos, "Tx/FH interrupt:\t\t %u\n",
2734 		isr_stats->tx);
2735 
2736 	pos += scnprintf(buf + pos, bufsz - pos, "Unexpected INTA:\t\t %u\n",
2737 		isr_stats->unhandled);
2738 
2739 	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2740 	kfree(buf);
2741 	return ret;
2742 }
2743 
2744 static ssize_t iwl_dbgfs_interrupt_write(struct file *file,
2745 					 const char __user *user_buf,
2746 					 size_t count, loff_t *ppos)
2747 {
2748 	struct iwl_trans *trans = file->private_data;
2749 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2750 	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
2751 	u32 reset_flag;
2752 	int ret;
2753 
2754 	ret = kstrtou32_from_user(user_buf, count, 16, &reset_flag);
2755 	if (ret)
2756 		return ret;
2757 	if (reset_flag == 0)
2758 		memset(isr_stats, 0, sizeof(*isr_stats));
2759 
2760 	return count;
2761 }
2762 
2763 static ssize_t iwl_dbgfs_csr_write(struct file *file,
2764 				   const char __user *user_buf,
2765 				   size_t count, loff_t *ppos)
2766 {
2767 	struct iwl_trans *trans = file->private_data;
2768 
2769 	iwl_pcie_dump_csr(trans);
2770 
2771 	return count;
2772 }
2773 
2774 static ssize_t iwl_dbgfs_fh_reg_read(struct file *file,
2775 				     char __user *user_buf,
2776 				     size_t count, loff_t *ppos)
2777 {
2778 	struct iwl_trans *trans = file->private_data;
2779 	char *buf = NULL;
2780 	ssize_t ret;
2781 
2782 	ret = iwl_dump_fh(trans, &buf);
2783 	if (ret < 0)
2784 		return ret;
2785 	if (!buf)
2786 		return -EINVAL;
2787 	ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret);
2788 	kfree(buf);
2789 	return ret;
2790 }
2791 
2792 static ssize_t iwl_dbgfs_rfkill_read(struct file *file,
2793 				     char __user *user_buf,
2794 				     size_t count, loff_t *ppos)
2795 {
2796 	struct iwl_trans *trans = file->private_data;
2797 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2798 	char buf[100];
2799 	int pos;
2800 
2801 	pos = scnprintf(buf, sizeof(buf), "debug: %d\nhw: %d\n",
2802 			trans_pcie->debug_rfkill,
2803 			!(iwl_read32(trans, CSR_GP_CNTRL) &
2804 				CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW));
2805 
2806 	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2807 }
2808 
2809 static ssize_t iwl_dbgfs_rfkill_write(struct file *file,
2810 				      const char __user *user_buf,
2811 				      size_t count, loff_t *ppos)
2812 {
2813 	struct iwl_trans *trans = file->private_data;
2814 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2815 	bool new_value;
2816 	int ret;
2817 
2818 	ret = kstrtobool_from_user(user_buf, count, &new_value);
2819 	if (ret)
2820 		return ret;
2821 	if (new_value == trans_pcie->debug_rfkill)
2822 		return count;
2823 	IWL_WARN(trans, "changing debug rfkill %d->%d\n",
2824 		 trans_pcie->debug_rfkill, new_value);
2825 	trans_pcie->debug_rfkill = new_value;
2826 	iwl_pcie_handle_rfkill_irq(trans);
2827 
2828 	return count;
2829 }
2830 
2831 static int iwl_dbgfs_monitor_data_open(struct inode *inode,
2832 				       struct file *file)
2833 {
2834 	struct iwl_trans *trans = inode->i_private;
2835 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2836 
2837 	if (!trans->dbg.dest_tlv ||
2838 	    trans->dbg.dest_tlv->monitor_mode != EXTERNAL_MODE) {
2839 		IWL_ERR(trans, "Debug destination is not set to DRAM\n");
2840 		return -ENOENT;
2841 	}
2842 
2843 	if (trans_pcie->fw_mon_data.state != IWL_FW_MON_DBGFS_STATE_CLOSED)
2844 		return -EBUSY;
2845 
2846 	trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_OPEN;
2847 	return simple_open(inode, file);
2848 }
2849 
2850 static int iwl_dbgfs_monitor_data_release(struct inode *inode,
2851 					  struct file *file)
2852 {
2853 	struct iwl_trans_pcie *trans_pcie =
2854 		IWL_TRANS_GET_PCIE_TRANS(inode->i_private);
2855 
2856 	if (trans_pcie->fw_mon_data.state == IWL_FW_MON_DBGFS_STATE_OPEN)
2857 		trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED;
2858 	return 0;
2859 }
2860 
2861 static bool iwl_write_to_user_buf(char __user *user_buf, ssize_t count,
2862 				  void *buf, ssize_t *size,
2863 				  ssize_t *bytes_copied)
2864 {
2865 	ssize_t buf_size_left = count - *bytes_copied;
2866 
2867 	buf_size_left = buf_size_left - (buf_size_left % sizeof(u32));
2868 	if (*size > buf_size_left)
2869 		*size = buf_size_left;
2870 
2871 	*size -= copy_to_user(user_buf, buf, *size);
2872 	*bytes_copied += *size;
2873 
2874 	if (buf_size_left == *size)
2875 		return true;
2876 	return false;
2877 }
2878 
2879 static ssize_t iwl_dbgfs_monitor_data_read(struct file *file,
2880 					   char __user *user_buf,
2881 					   size_t count, loff_t *ppos)
2882 {
2883 	struct iwl_trans *trans = file->private_data;
2884 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2885 	u8 *cpu_addr = (void *)trans->dbg.fw_mon.block, *curr_buf;
2886 	struct cont_rec *data = &trans_pcie->fw_mon_data;
2887 	u32 write_ptr_addr, wrap_cnt_addr, write_ptr, wrap_cnt;
2888 	ssize_t size, bytes_copied = 0;
2889 	bool b_full;
2890 
2891 	if (trans->dbg.dest_tlv) {
2892 		write_ptr_addr =
2893 			le32_to_cpu(trans->dbg.dest_tlv->write_ptr_reg);
2894 		wrap_cnt_addr = le32_to_cpu(trans->dbg.dest_tlv->wrap_count);
2895 	} else {
2896 		write_ptr_addr = MON_BUFF_WRPTR;
2897 		wrap_cnt_addr = MON_BUFF_CYCLE_CNT;
2898 	}
2899 
2900 	if (unlikely(!trans->dbg.rec_on))
2901 		return 0;
2902 
2903 	mutex_lock(&data->mutex);
2904 	if (data->state ==
2905 	    IWL_FW_MON_DBGFS_STATE_DISABLED) {
2906 		mutex_unlock(&data->mutex);
2907 		return 0;
2908 	}
2909 
2910 	/* write_ptr position in bytes rather then DW */
2911 	write_ptr = iwl_read_prph(trans, write_ptr_addr) * sizeof(u32);
2912 	wrap_cnt = iwl_read_prph(trans, wrap_cnt_addr);
2913 
2914 	if (data->prev_wrap_cnt == wrap_cnt) {
2915 		size = write_ptr - data->prev_wr_ptr;
2916 		curr_buf = cpu_addr + data->prev_wr_ptr;
2917 		b_full = iwl_write_to_user_buf(user_buf, count,
2918 					       curr_buf, &size,
2919 					       &bytes_copied);
2920 		data->prev_wr_ptr += size;
2921 
2922 	} else if (data->prev_wrap_cnt == wrap_cnt - 1 &&
2923 		   write_ptr < data->prev_wr_ptr) {
2924 		size = trans->dbg.fw_mon.size - data->prev_wr_ptr;
2925 		curr_buf = cpu_addr + data->prev_wr_ptr;
2926 		b_full = iwl_write_to_user_buf(user_buf, count,
2927 					       curr_buf, &size,
2928 					       &bytes_copied);
2929 		data->prev_wr_ptr += size;
2930 
2931 		if (!b_full) {
2932 			size = write_ptr;
2933 			b_full = iwl_write_to_user_buf(user_buf, count,
2934 						       cpu_addr, &size,
2935 						       &bytes_copied);
2936 			data->prev_wr_ptr = size;
2937 			data->prev_wrap_cnt++;
2938 		}
2939 	} else {
2940 		if (data->prev_wrap_cnt == wrap_cnt - 1 &&
2941 		    write_ptr > data->prev_wr_ptr)
2942 			IWL_WARN(trans,
2943 				 "write pointer passed previous write pointer, start copying from the beginning\n");
2944 		else if (!unlikely(data->prev_wrap_cnt == 0 &&
2945 				   data->prev_wr_ptr == 0))
2946 			IWL_WARN(trans,
2947 				 "monitor data is out of sync, start copying from the beginning\n");
2948 
2949 		size = write_ptr;
2950 		b_full = iwl_write_to_user_buf(user_buf, count,
2951 					       cpu_addr, &size,
2952 					       &bytes_copied);
2953 		data->prev_wr_ptr = size;
2954 		data->prev_wrap_cnt = wrap_cnt;
2955 	}
2956 
2957 	mutex_unlock(&data->mutex);
2958 
2959 	return bytes_copied;
2960 }
2961 
2962 static ssize_t iwl_dbgfs_rf_read(struct file *file,
2963 				 char __user *user_buf,
2964 				 size_t count, loff_t *ppos)
2965 {
2966 	struct iwl_trans *trans = file->private_data;
2967 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2968 
2969 	if (!trans_pcie->rf_name[0])
2970 		return -ENODEV;
2971 
2972 	return simple_read_from_buffer(user_buf, count, ppos,
2973 				       trans_pcie->rf_name,
2974 				       strlen(trans_pcie->rf_name));
2975 }
2976 
2977 DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
2978 DEBUGFS_READ_FILE_OPS(fh_reg);
2979 DEBUGFS_READ_FILE_OPS(rx_queue);
2980 DEBUGFS_WRITE_FILE_OPS(csr);
2981 DEBUGFS_READ_WRITE_FILE_OPS(rfkill);
2982 DEBUGFS_READ_FILE_OPS(rf);
2983 
2984 static const struct file_operations iwl_dbgfs_tx_queue_ops = {
2985 	.owner = THIS_MODULE,
2986 	.open = iwl_dbgfs_tx_queue_open,
2987 	.read = seq_read,
2988 	.llseek = seq_lseek,
2989 	.release = seq_release_private,
2990 };
2991 
2992 static const struct file_operations iwl_dbgfs_monitor_data_ops = {
2993 	.read = iwl_dbgfs_monitor_data_read,
2994 	.open = iwl_dbgfs_monitor_data_open,
2995 	.release = iwl_dbgfs_monitor_data_release,
2996 };
2997 
2998 /* Create the debugfs files and directories */
2999 void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
3000 {
3001 	struct dentry *dir = trans->dbgfs_dir;
3002 
3003 	DEBUGFS_ADD_FILE(rx_queue, dir, 0400);
3004 	DEBUGFS_ADD_FILE(tx_queue, dir, 0400);
3005 	DEBUGFS_ADD_FILE(interrupt, dir, 0600);
3006 	DEBUGFS_ADD_FILE(csr, dir, 0200);
3007 	DEBUGFS_ADD_FILE(fh_reg, dir, 0400);
3008 	DEBUGFS_ADD_FILE(rfkill, dir, 0600);
3009 	DEBUGFS_ADD_FILE(monitor_data, dir, 0400);
3010 	DEBUGFS_ADD_FILE(rf, dir, 0400);
3011 }
3012 
3013 static void iwl_trans_pcie_debugfs_cleanup(struct iwl_trans *trans)
3014 {
3015 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3016 	struct cont_rec *data = &trans_pcie->fw_mon_data;
3017 
3018 	mutex_lock(&data->mutex);
3019 	data->state = IWL_FW_MON_DBGFS_STATE_DISABLED;
3020 	mutex_unlock(&data->mutex);
3021 }
3022 #endif /*CONFIG_IWLWIFI_DEBUGFS */
3023 
3024 static u32 iwl_trans_pcie_get_cmdlen(struct iwl_trans *trans, void *tfd)
3025 {
3026 	u32 cmdlen = 0;
3027 	int i;
3028 
3029 	for (i = 0; i < trans->txqs.tfd.max_tbs; i++)
3030 		cmdlen += iwl_txq_gen1_tfd_tb_get_len(trans, tfd, i);
3031 
3032 	return cmdlen;
3033 }
3034 
3035 static u32 iwl_trans_pcie_dump_rbs(struct iwl_trans *trans,
3036 				   struct iwl_fw_error_dump_data **data,
3037 				   int allocated_rb_nums)
3038 {
3039 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3040 	int max_len = trans_pcie->rx_buf_bytes;
3041 	/* Dump RBs is supported only for pre-9000 devices (1 queue) */
3042 	struct iwl_rxq *rxq = &trans_pcie->rxq[0];
3043 	u32 i, r, j, rb_len = 0;
3044 
3045 	spin_lock(&rxq->lock);
3046 
3047 	r = le16_to_cpu(iwl_get_closed_rb_stts(trans, rxq)) & 0x0FFF;
3048 
3049 	for (i = rxq->read, j = 0;
3050 	     i != r && j < allocated_rb_nums;
3051 	     i = (i + 1) & RX_QUEUE_MASK, j++) {
3052 		struct iwl_rx_mem_buffer *rxb = rxq->queue[i];
3053 		struct iwl_fw_error_dump_rb *rb;
3054 
3055 		dma_sync_single_for_cpu(trans->dev, rxb->page_dma,
3056 					max_len, DMA_FROM_DEVICE);
3057 
3058 		rb_len += sizeof(**data) + sizeof(*rb) + max_len;
3059 
3060 		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RB);
3061 		(*data)->len = cpu_to_le32(sizeof(*rb) + max_len);
3062 		rb = (void *)(*data)->data;
3063 		rb->index = cpu_to_le32(i);
3064 		memcpy(rb->data, page_address(rxb->page), max_len);
3065 
3066 		*data = iwl_fw_error_next_data(*data);
3067 	}
3068 
3069 	spin_unlock(&rxq->lock);
3070 
3071 	return rb_len;
3072 }
3073 #define IWL_CSR_TO_DUMP (0x250)
3074 
3075 static u32 iwl_trans_pcie_dump_csr(struct iwl_trans *trans,
3076 				   struct iwl_fw_error_dump_data **data)
3077 {
3078 	u32 csr_len = sizeof(**data) + IWL_CSR_TO_DUMP;
3079 	__le32 *val;
3080 	int i;
3081 
3082 	(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_CSR);
3083 	(*data)->len = cpu_to_le32(IWL_CSR_TO_DUMP);
3084 	val = (void *)(*data)->data;
3085 
3086 	for (i = 0; i < IWL_CSR_TO_DUMP; i += 4)
3087 		*val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
3088 
3089 	*data = iwl_fw_error_next_data(*data);
3090 
3091 	return csr_len;
3092 }
3093 
3094 static u32 iwl_trans_pcie_fh_regs_dump(struct iwl_trans *trans,
3095 				       struct iwl_fw_error_dump_data **data)
3096 {
3097 	u32 fh_regs_len = FH_MEM_UPPER_BOUND - FH_MEM_LOWER_BOUND;
3098 	__le32 *val;
3099 	int i;
3100 
3101 	if (!iwl_trans_grab_nic_access(trans))
3102 		return 0;
3103 
3104 	(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FH_REGS);
3105 	(*data)->len = cpu_to_le32(fh_regs_len);
3106 	val = (void *)(*data)->data;
3107 
3108 	if (!trans->trans_cfg->gen2)
3109 		for (i = FH_MEM_LOWER_BOUND; i < FH_MEM_UPPER_BOUND;
3110 		     i += sizeof(u32))
3111 			*val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
3112 	else
3113 		for (i = iwl_umac_prph(trans, FH_MEM_LOWER_BOUND_GEN2);
3114 		     i < iwl_umac_prph(trans, FH_MEM_UPPER_BOUND_GEN2);
3115 		     i += sizeof(u32))
3116 			*val++ = cpu_to_le32(iwl_trans_pcie_read_prph(trans,
3117 								      i));
3118 
3119 	iwl_trans_release_nic_access(trans);
3120 
3121 	*data = iwl_fw_error_next_data(*data);
3122 
3123 	return sizeof(**data) + fh_regs_len;
3124 }
3125 
3126 static u32
3127 iwl_trans_pci_dump_marbh_monitor(struct iwl_trans *trans,
3128 				 struct iwl_fw_error_dump_fw_mon *fw_mon_data,
3129 				 u32 monitor_len)
3130 {
3131 	u32 buf_size_in_dwords = (monitor_len >> 2);
3132 	u32 *buffer = (u32 *)fw_mon_data->data;
3133 	u32 i;
3134 
3135 	if (!iwl_trans_grab_nic_access(trans))
3136 		return 0;
3137 
3138 	iwl_write_umac_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x1);
3139 	for (i = 0; i < buf_size_in_dwords; i++)
3140 		buffer[i] = iwl_read_umac_prph_no_grab(trans,
3141 						       MON_DMARB_RD_DATA_ADDR);
3142 	iwl_write_umac_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x0);
3143 
3144 	iwl_trans_release_nic_access(trans);
3145 
3146 	return monitor_len;
3147 }
3148 
3149 static void
3150 iwl_trans_pcie_dump_pointers(struct iwl_trans *trans,
3151 			     struct iwl_fw_error_dump_fw_mon *fw_mon_data)
3152 {
3153 	u32 base, base_high, write_ptr, write_ptr_val, wrap_cnt;
3154 
3155 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
3156 		base = DBGC_CUR_DBGBUF_BASE_ADDR_LSB;
3157 		base_high = DBGC_CUR_DBGBUF_BASE_ADDR_MSB;
3158 		write_ptr = DBGC_CUR_DBGBUF_STATUS;
3159 		wrap_cnt = DBGC_DBGBUF_WRAP_AROUND;
3160 	} else if (trans->dbg.dest_tlv) {
3161 		write_ptr = le32_to_cpu(trans->dbg.dest_tlv->write_ptr_reg);
3162 		wrap_cnt = le32_to_cpu(trans->dbg.dest_tlv->wrap_count);
3163 		base = le32_to_cpu(trans->dbg.dest_tlv->base_reg);
3164 	} else {
3165 		base = MON_BUFF_BASE_ADDR;
3166 		write_ptr = MON_BUFF_WRPTR;
3167 		wrap_cnt = MON_BUFF_CYCLE_CNT;
3168 	}
3169 
3170 	write_ptr_val = iwl_read_prph(trans, write_ptr);
3171 	fw_mon_data->fw_mon_cycle_cnt =
3172 		cpu_to_le32(iwl_read_prph(trans, wrap_cnt));
3173 	fw_mon_data->fw_mon_base_ptr =
3174 		cpu_to_le32(iwl_read_prph(trans, base));
3175 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
3176 		fw_mon_data->fw_mon_base_high_ptr =
3177 			cpu_to_le32(iwl_read_prph(trans, base_high));
3178 		write_ptr_val &= DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK;
3179 		/* convert wrtPtr to DWs, to align with all HWs */
3180 		write_ptr_val >>= 2;
3181 	}
3182 	fw_mon_data->fw_mon_wr_ptr = cpu_to_le32(write_ptr_val);
3183 }
3184 
3185 static u32
3186 iwl_trans_pcie_dump_monitor(struct iwl_trans *trans,
3187 			    struct iwl_fw_error_dump_data **data,
3188 			    u32 monitor_len)
3189 {
3190 	struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
3191 	u32 len = 0;
3192 
3193 	if (trans->dbg.dest_tlv ||
3194 	    (fw_mon->size &&
3195 	     (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000 ||
3196 	      trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210))) {
3197 		struct iwl_fw_error_dump_fw_mon *fw_mon_data;
3198 
3199 		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FW_MONITOR);
3200 		fw_mon_data = (void *)(*data)->data;
3201 
3202 		iwl_trans_pcie_dump_pointers(trans, fw_mon_data);
3203 
3204 		len += sizeof(**data) + sizeof(*fw_mon_data);
3205 		if (fw_mon->size) {
3206 			memcpy(fw_mon_data->data, fw_mon->block, fw_mon->size);
3207 			monitor_len = fw_mon->size;
3208 		} else if (trans->dbg.dest_tlv->monitor_mode == SMEM_MODE) {
3209 			u32 base = le32_to_cpu(fw_mon_data->fw_mon_base_ptr);
3210 			/*
3211 			 * Update pointers to reflect actual values after
3212 			 * shifting
3213 			 */
3214 			if (trans->dbg.dest_tlv->version) {
3215 				base = (iwl_read_prph(trans, base) &
3216 					IWL_LDBG_M2S_BUF_BA_MSK) <<
3217 				       trans->dbg.dest_tlv->base_shift;
3218 				base *= IWL_M2S_UNIT_SIZE;
3219 				base += trans->cfg->smem_offset;
3220 			} else {
3221 				base = iwl_read_prph(trans, base) <<
3222 				       trans->dbg.dest_tlv->base_shift;
3223 			}
3224 
3225 			iwl_trans_read_mem(trans, base, fw_mon_data->data,
3226 					   monitor_len / sizeof(u32));
3227 		} else if (trans->dbg.dest_tlv->monitor_mode == MARBH_MODE) {
3228 			monitor_len =
3229 				iwl_trans_pci_dump_marbh_monitor(trans,
3230 								 fw_mon_data,
3231 								 monitor_len);
3232 		} else {
3233 			/* Didn't match anything - output no monitor data */
3234 			monitor_len = 0;
3235 		}
3236 
3237 		len += monitor_len;
3238 		(*data)->len = cpu_to_le32(monitor_len + sizeof(*fw_mon_data));
3239 	}
3240 
3241 	return len;
3242 }
3243 
3244 static int iwl_trans_get_fw_monitor_len(struct iwl_trans *trans, u32 *len)
3245 {
3246 	if (trans->dbg.fw_mon.size) {
3247 		*len += sizeof(struct iwl_fw_error_dump_data) +
3248 			sizeof(struct iwl_fw_error_dump_fw_mon) +
3249 			trans->dbg.fw_mon.size;
3250 		return trans->dbg.fw_mon.size;
3251 	} else if (trans->dbg.dest_tlv) {
3252 		u32 base, end, cfg_reg, monitor_len;
3253 
3254 		if (trans->dbg.dest_tlv->version == 1) {
3255 			cfg_reg = le32_to_cpu(trans->dbg.dest_tlv->base_reg);
3256 			cfg_reg = iwl_read_prph(trans, cfg_reg);
3257 			base = (cfg_reg & IWL_LDBG_M2S_BUF_BA_MSK) <<
3258 				trans->dbg.dest_tlv->base_shift;
3259 			base *= IWL_M2S_UNIT_SIZE;
3260 			base += trans->cfg->smem_offset;
3261 
3262 			monitor_len =
3263 				(cfg_reg & IWL_LDBG_M2S_BUF_SIZE_MSK) >>
3264 				trans->dbg.dest_tlv->end_shift;
3265 			monitor_len *= IWL_M2S_UNIT_SIZE;
3266 		} else {
3267 			base = le32_to_cpu(trans->dbg.dest_tlv->base_reg);
3268 			end = le32_to_cpu(trans->dbg.dest_tlv->end_reg);
3269 
3270 			base = iwl_read_prph(trans, base) <<
3271 			       trans->dbg.dest_tlv->base_shift;
3272 			end = iwl_read_prph(trans, end) <<
3273 			      trans->dbg.dest_tlv->end_shift;
3274 
3275 			/* Make "end" point to the actual end */
3276 			if (trans->trans_cfg->device_family >=
3277 			    IWL_DEVICE_FAMILY_8000 ||
3278 			    trans->dbg.dest_tlv->monitor_mode == MARBH_MODE)
3279 				end += (1 << trans->dbg.dest_tlv->end_shift);
3280 			monitor_len = end - base;
3281 		}
3282 		*len += sizeof(struct iwl_fw_error_dump_data) +
3283 			sizeof(struct iwl_fw_error_dump_fw_mon) +
3284 			monitor_len;
3285 		return monitor_len;
3286 	}
3287 	return 0;
3288 }
3289 
3290 static struct iwl_trans_dump_data *
3291 iwl_trans_pcie_dump_data(struct iwl_trans *trans,
3292 			 u32 dump_mask,
3293 			 const struct iwl_dump_sanitize_ops *sanitize_ops,
3294 			 void *sanitize_ctx)
3295 {
3296 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3297 	struct iwl_fw_error_dump_data *data;
3298 	struct iwl_txq *cmdq = trans->txqs.txq[trans->txqs.cmd.q_id];
3299 	struct iwl_fw_error_dump_txcmd *txcmd;
3300 	struct iwl_trans_dump_data *dump_data;
3301 	u32 len, num_rbs = 0, monitor_len = 0;
3302 	int i, ptr;
3303 	bool dump_rbs = test_bit(STATUS_FW_ERROR, &trans->status) &&
3304 			!trans->trans_cfg->mq_rx_supported &&
3305 			dump_mask & BIT(IWL_FW_ERROR_DUMP_RB);
3306 
3307 	if (!dump_mask)
3308 		return NULL;
3309 
3310 	/* transport dump header */
3311 	len = sizeof(*dump_data);
3312 
3313 	/* host commands */
3314 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_TXCMD) && cmdq)
3315 		len += sizeof(*data) +
3316 			cmdq->n_window * (sizeof(*txcmd) +
3317 					  TFD_MAX_PAYLOAD_SIZE);
3318 
3319 	/* FW monitor */
3320 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FW_MONITOR))
3321 		monitor_len = iwl_trans_get_fw_monitor_len(trans, &len);
3322 
3323 	/* CSR registers */
3324 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR))
3325 		len += sizeof(*data) + IWL_CSR_TO_DUMP;
3326 
3327 	/* FH registers */
3328 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS)) {
3329 		if (trans->trans_cfg->gen2)
3330 			len += sizeof(*data) +
3331 			       (iwl_umac_prph(trans, FH_MEM_UPPER_BOUND_GEN2) -
3332 				iwl_umac_prph(trans, FH_MEM_LOWER_BOUND_GEN2));
3333 		else
3334 			len += sizeof(*data) +
3335 			       (FH_MEM_UPPER_BOUND -
3336 				FH_MEM_LOWER_BOUND);
3337 	}
3338 
3339 	if (dump_rbs) {
3340 		/* Dump RBs is supported only for pre-9000 devices (1 queue) */
3341 		struct iwl_rxq *rxq = &trans_pcie->rxq[0];
3342 		/* RBs */
3343 		num_rbs =
3344 			le16_to_cpu(iwl_get_closed_rb_stts(trans, rxq))
3345 			& 0x0FFF;
3346 		num_rbs = (num_rbs - rxq->read) & RX_QUEUE_MASK;
3347 		len += num_rbs * (sizeof(*data) +
3348 				  sizeof(struct iwl_fw_error_dump_rb) +
3349 				  (PAGE_SIZE << trans_pcie->rx_page_order));
3350 	}
3351 
3352 	/* Paged memory for gen2 HW */
3353 	if (trans->trans_cfg->gen2 && dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING))
3354 		for (i = 0; i < trans->init_dram.paging_cnt; i++)
3355 			len += sizeof(*data) +
3356 			       sizeof(struct iwl_fw_error_dump_paging) +
3357 			       trans->init_dram.paging[i].size;
3358 
3359 	dump_data = vzalloc(len);
3360 	if (!dump_data)
3361 		return NULL;
3362 
3363 	len = 0;
3364 	data = (void *)dump_data->data;
3365 
3366 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_TXCMD) && cmdq) {
3367 		u16 tfd_size = trans->txqs.tfd.size;
3368 
3369 		data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXCMD);
3370 		txcmd = (void *)data->data;
3371 		spin_lock_bh(&cmdq->lock);
3372 		ptr = cmdq->write_ptr;
3373 		for (i = 0; i < cmdq->n_window; i++) {
3374 			u8 idx = iwl_txq_get_cmd_index(cmdq, ptr);
3375 			u8 tfdidx;
3376 			u32 caplen, cmdlen;
3377 
3378 			if (trans->trans_cfg->use_tfh)
3379 				tfdidx = idx;
3380 			else
3381 				tfdidx = ptr;
3382 
3383 			cmdlen = iwl_trans_pcie_get_cmdlen(trans,
3384 							   (u8 *)cmdq->tfds +
3385 							   tfd_size * tfdidx);
3386 			caplen = min_t(u32, TFD_MAX_PAYLOAD_SIZE, cmdlen);
3387 
3388 			if (cmdlen) {
3389 				len += sizeof(*txcmd) + caplen;
3390 				txcmd->cmdlen = cpu_to_le32(cmdlen);
3391 				txcmd->caplen = cpu_to_le32(caplen);
3392 				memcpy(txcmd->data, cmdq->entries[idx].cmd,
3393 				       caplen);
3394 				if (sanitize_ops && sanitize_ops->frob_hcmd)
3395 					sanitize_ops->frob_hcmd(sanitize_ctx,
3396 								txcmd->data,
3397 								caplen);
3398 				txcmd = (void *)((u8 *)txcmd->data + caplen);
3399 			}
3400 
3401 			ptr = iwl_txq_dec_wrap(trans, ptr);
3402 		}
3403 		spin_unlock_bh(&cmdq->lock);
3404 
3405 		data->len = cpu_to_le32(len);
3406 		len += sizeof(*data);
3407 		data = iwl_fw_error_next_data(data);
3408 	}
3409 
3410 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR))
3411 		len += iwl_trans_pcie_dump_csr(trans, &data);
3412 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS))
3413 		len += iwl_trans_pcie_fh_regs_dump(trans, &data);
3414 	if (dump_rbs)
3415 		len += iwl_trans_pcie_dump_rbs(trans, &data, num_rbs);
3416 
3417 	/* Paged memory for gen2 HW */
3418 	if (trans->trans_cfg->gen2 &&
3419 	    dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING)) {
3420 		for (i = 0; i < trans->init_dram.paging_cnt; i++) {
3421 			struct iwl_fw_error_dump_paging *paging;
3422 			u32 page_len = trans->init_dram.paging[i].size;
3423 
3424 			data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
3425 			data->len = cpu_to_le32(sizeof(*paging) + page_len);
3426 			paging = (void *)data->data;
3427 			paging->index = cpu_to_le32(i);
3428 			memcpy(paging->data,
3429 			       trans->init_dram.paging[i].block, page_len);
3430 			data = iwl_fw_error_next_data(data);
3431 
3432 			len += sizeof(*data) + sizeof(*paging) + page_len;
3433 		}
3434 	}
3435 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FW_MONITOR))
3436 		len += iwl_trans_pcie_dump_monitor(trans, &data, monitor_len);
3437 
3438 	dump_data->len = len;
3439 
3440 	return dump_data;
3441 }
3442 
3443 static void iwl_trans_pci_interrupts(struct iwl_trans *trans, bool enable)
3444 {
3445 	if (enable)
3446 		iwl_enable_interrupts(trans);
3447 	else
3448 		iwl_disable_interrupts(trans);
3449 }
3450 
3451 static void iwl_trans_pcie_sync_nmi(struct iwl_trans *trans)
3452 {
3453 	u32 inta_addr, sw_err_bit;
3454 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3455 
3456 	if (trans_pcie->msix_enabled) {
3457 		inta_addr = CSR_MSIX_HW_INT_CAUSES_AD;
3458 		if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
3459 			sw_err_bit = MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ;
3460 		else
3461 			sw_err_bit = MSIX_HW_INT_CAUSES_REG_SW_ERR;
3462 	} else {
3463 		inta_addr = CSR_INT;
3464 		sw_err_bit = CSR_INT_BIT_SW_ERR;
3465 	}
3466 
3467 	iwl_trans_sync_nmi_with_addr(trans, inta_addr, sw_err_bit);
3468 }
3469 
3470 #define IWL_TRANS_COMMON_OPS						\
3471 	.op_mode_leave = iwl_trans_pcie_op_mode_leave,			\
3472 	.write8 = iwl_trans_pcie_write8,				\
3473 	.write32 = iwl_trans_pcie_write32,				\
3474 	.read32 = iwl_trans_pcie_read32,				\
3475 	.read_prph = iwl_trans_pcie_read_prph,				\
3476 	.write_prph = iwl_trans_pcie_write_prph,			\
3477 	.read_mem = iwl_trans_pcie_read_mem,				\
3478 	.write_mem = iwl_trans_pcie_write_mem,				\
3479 	.read_config32 = iwl_trans_pcie_read_config32,			\
3480 	.configure = iwl_trans_pcie_configure,				\
3481 	.set_pmi = iwl_trans_pcie_set_pmi,				\
3482 	.sw_reset = iwl_trans_pcie_sw_reset,				\
3483 	.grab_nic_access = iwl_trans_pcie_grab_nic_access,		\
3484 	.release_nic_access = iwl_trans_pcie_release_nic_access,	\
3485 	.set_bits_mask = iwl_trans_pcie_set_bits_mask,			\
3486 	.dump_data = iwl_trans_pcie_dump_data,				\
3487 	.d3_suspend = iwl_trans_pcie_d3_suspend,			\
3488 	.d3_resume = iwl_trans_pcie_d3_resume,				\
3489 	.interrupts = iwl_trans_pci_interrupts,				\
3490 	.sync_nmi = iwl_trans_pcie_sync_nmi,				\
3491 	.imr_dma_data = iwl_trans_pcie_copy_imr				\
3492 
3493 static const struct iwl_trans_ops trans_ops_pcie = {
3494 	IWL_TRANS_COMMON_OPS,
3495 	.start_hw = iwl_trans_pcie_start_hw,
3496 	.fw_alive = iwl_trans_pcie_fw_alive,
3497 	.start_fw = iwl_trans_pcie_start_fw,
3498 	.stop_device = iwl_trans_pcie_stop_device,
3499 
3500 	.send_cmd = iwl_pcie_enqueue_hcmd,
3501 
3502 	.tx = iwl_trans_pcie_tx,
3503 	.reclaim = iwl_txq_reclaim,
3504 
3505 	.txq_disable = iwl_trans_pcie_txq_disable,
3506 	.txq_enable = iwl_trans_pcie_txq_enable,
3507 
3508 	.txq_set_shared_mode = iwl_trans_pcie_txq_set_shared_mode,
3509 
3510 	.wait_tx_queues_empty = iwl_trans_pcie_wait_txqs_empty,
3511 
3512 	.freeze_txq_timer = iwl_trans_txq_freeze_timer,
3513 	.block_txq_ptrs = iwl_trans_pcie_block_txq_ptrs,
3514 #ifdef CONFIG_IWLWIFI_DEBUGFS
3515 	.debugfs_cleanup = iwl_trans_pcie_debugfs_cleanup,
3516 #endif
3517 };
3518 
3519 static const struct iwl_trans_ops trans_ops_pcie_gen2 = {
3520 	IWL_TRANS_COMMON_OPS,
3521 	.start_hw = iwl_trans_pcie_start_hw,
3522 	.fw_alive = iwl_trans_pcie_gen2_fw_alive,
3523 	.start_fw = iwl_trans_pcie_gen2_start_fw,
3524 	.stop_device = iwl_trans_pcie_gen2_stop_device,
3525 
3526 	.send_cmd = iwl_pcie_gen2_enqueue_hcmd,
3527 
3528 	.tx = iwl_txq_gen2_tx,
3529 	.reclaim = iwl_txq_reclaim,
3530 
3531 	.set_q_ptrs = iwl_txq_set_q_ptrs,
3532 
3533 	.txq_alloc = iwl_txq_dyn_alloc,
3534 	.txq_free = iwl_txq_dyn_free,
3535 	.wait_txq_empty = iwl_trans_pcie_wait_txq_empty,
3536 	.rxq_dma_data = iwl_trans_pcie_rxq_dma_data,
3537 	.set_pnvm = iwl_trans_pcie_ctx_info_gen3_set_pnvm,
3538 	.set_reduce_power = iwl_trans_pcie_ctx_info_gen3_set_reduce_power,
3539 #ifdef CONFIG_IWLWIFI_DEBUGFS
3540 	.debugfs_cleanup = iwl_trans_pcie_debugfs_cleanup,
3541 #endif
3542 };
3543 
3544 struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
3545 			       const struct pci_device_id *ent,
3546 			       const struct iwl_cfg_trans_params *cfg_trans)
3547 {
3548 	struct iwl_trans_pcie *trans_pcie;
3549 	struct iwl_trans *trans;
3550 	int ret, addr_size;
3551 	const struct iwl_trans_ops *ops = &trans_ops_pcie_gen2;
3552 	void __iomem * const *table;
3553 
3554 	if (!cfg_trans->gen2)
3555 		ops = &trans_ops_pcie;
3556 
3557 	ret = pcim_enable_device(pdev);
3558 	if (ret)
3559 		return ERR_PTR(ret);
3560 
3561 	trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie), &pdev->dev, ops,
3562 				cfg_trans);
3563 	if (!trans)
3564 		return ERR_PTR(-ENOMEM);
3565 
3566 	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3567 
3568 	trans_pcie->trans = trans;
3569 	trans_pcie->opmode_down = true;
3570 	spin_lock_init(&trans_pcie->irq_lock);
3571 	spin_lock_init(&trans_pcie->reg_lock);
3572 	spin_lock_init(&trans_pcie->alloc_page_lock);
3573 	mutex_init(&trans_pcie->mutex);
3574 	init_waitqueue_head(&trans_pcie->ucode_write_waitq);
3575 	init_waitqueue_head(&trans_pcie->fw_reset_waitq);
3576 	init_waitqueue_head(&trans_pcie->imr_waitq);
3577 
3578 	trans_pcie->rba.alloc_wq = alloc_workqueue("rb_allocator",
3579 						   WQ_HIGHPRI | WQ_UNBOUND, 0);
3580 	if (!trans_pcie->rba.alloc_wq) {
3581 		ret = -ENOMEM;
3582 		goto out_free_trans;
3583 	}
3584 	INIT_WORK(&trans_pcie->rba.rx_alloc, iwl_pcie_rx_allocator_work);
3585 
3586 	trans_pcie->debug_rfkill = -1;
3587 
3588 	if (!cfg_trans->base_params->pcie_l1_allowed) {
3589 		/*
3590 		 * W/A - seems to solve weird behavior. We need to remove this
3591 		 * if we don't want to stay in L1 all the time. This wastes a
3592 		 * lot of power.
3593 		 */
3594 		pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S |
3595 				       PCIE_LINK_STATE_L1 |
3596 				       PCIE_LINK_STATE_CLKPM);
3597 	}
3598 
3599 	trans_pcie->def_rx_queue = 0;
3600 
3601 	pci_set_master(pdev);
3602 
3603 	addr_size = trans->txqs.tfd.addr_size;
3604 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(addr_size));
3605 	if (ret) {
3606 		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3607 		/* both attempts failed: */
3608 		if (ret) {
3609 			dev_err(&pdev->dev, "No suitable DMA available\n");
3610 			goto out_no_pci;
3611 		}
3612 	}
3613 
3614 	ret = pcim_iomap_regions_request_all(pdev, BIT(0), DRV_NAME);
3615 	if (ret) {
3616 		dev_err(&pdev->dev, "pcim_iomap_regions_request_all failed\n");
3617 		goto out_no_pci;
3618 	}
3619 
3620 	table = pcim_iomap_table(pdev);
3621 	if (!table) {
3622 		dev_err(&pdev->dev, "pcim_iomap_table failed\n");
3623 		ret = -ENOMEM;
3624 		goto out_no_pci;
3625 	}
3626 
3627 	trans_pcie->hw_base = table[0];
3628 	if (!trans_pcie->hw_base) {
3629 		dev_err(&pdev->dev, "couldn't find IO mem in first BAR\n");
3630 		ret = -ENODEV;
3631 		goto out_no_pci;
3632 	}
3633 
3634 	/* We disable the RETRY_TIMEOUT register (0x41) to keep
3635 	 * PCI Tx retries from interfering with C3 CPU state */
3636 	pci_write_config_byte(pdev, PCI_CFG_RETRY_TIMEOUT, 0x00);
3637 
3638 	trans_pcie->pci_dev = pdev;
3639 	iwl_disable_interrupts(trans);
3640 
3641 	trans->hw_rev = iwl_read32(trans, CSR_HW_REV);
3642 	if (trans->hw_rev == 0xffffffff) {
3643 		dev_err(&pdev->dev, "HW_REV=0xFFFFFFFF, PCI issues?\n");
3644 		ret = -EIO;
3645 		goto out_no_pci;
3646 	}
3647 
3648 	/*
3649 	 * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
3650 	 * changed, and now the revision step also includes bit 0-1 (no more
3651 	 * "dash" value). To keep hw_rev backwards compatible - we'll store it
3652 	 * in the old format.
3653 	 */
3654 	if (cfg_trans->device_family >= IWL_DEVICE_FAMILY_8000)
3655 		trans->hw_rev_step = trans->hw_rev & 0xF;
3656 	else
3657 		trans->hw_rev_step = (trans->hw_rev & 0xC) >> 2;
3658 
3659 	IWL_DEBUG_INFO(trans, "HW REV: 0x%0x\n", trans->hw_rev);
3660 
3661 	iwl_pcie_set_interrupt_capa(pdev, trans, cfg_trans);
3662 	trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
3663 	snprintf(trans->hw_id_str, sizeof(trans->hw_id_str),
3664 		 "PCI ID: 0x%04X:0x%04X", pdev->device, pdev->subsystem_device);
3665 
3666 	init_waitqueue_head(&trans_pcie->sx_waitq);
3667 
3668 
3669 	if (trans_pcie->msix_enabled) {
3670 		ret = iwl_pcie_init_msix_handler(pdev, trans_pcie);
3671 		if (ret)
3672 			goto out_no_pci;
3673 	 } else {
3674 		ret = iwl_pcie_alloc_ict(trans);
3675 		if (ret)
3676 			goto out_no_pci;
3677 
3678 		ret = devm_request_threaded_irq(&pdev->dev, pdev->irq,
3679 						iwl_pcie_isr,
3680 						iwl_pcie_irq_handler,
3681 						IRQF_SHARED, DRV_NAME, trans);
3682 		if (ret) {
3683 			IWL_ERR(trans, "Error allocating IRQ %d\n", pdev->irq);
3684 			goto out_free_ict;
3685 		}
3686 	 }
3687 
3688 #ifdef CONFIG_IWLWIFI_DEBUGFS
3689 	trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED;
3690 	mutex_init(&trans_pcie->fw_mon_data.mutex);
3691 #endif
3692 
3693 	iwl_dbg_tlv_init(trans);
3694 
3695 	return trans;
3696 
3697 out_free_ict:
3698 	iwl_pcie_free_ict(trans);
3699 out_no_pci:
3700 	destroy_workqueue(trans_pcie->rba.alloc_wq);
3701 out_free_trans:
3702 	iwl_trans_free(trans);
3703 	return ERR_PTR(ret);
3704 }
3705 
3706 void iwl_trans_pcie_copy_imr_fh(struct iwl_trans *trans,
3707 				u32 dst_addr, u64 src_addr, u32 byte_cnt)
3708 {
3709 	iwl_write_prph(trans, IMR_UREG_CHICK,
3710 		       iwl_read_prph(trans, IMR_UREG_CHICK) |
3711 		       IMR_UREG_CHICK_HALT_UMAC_PERMANENTLY_MSK);
3712 	iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_SRAM_ADDR, dst_addr);
3713 	iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_DRAM_ADDR_LSB,
3714 		       (u32)(src_addr & 0xFFFFFFFF));
3715 	iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_DRAM_ADDR_MSB,
3716 		       iwl_get_dma_hi_addr(src_addr));
3717 	iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_BC, byte_cnt);
3718 	iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_CTRL,
3719 		       IMR_TFH_SRV_DMA_CHNL0_CTRL_D2S_IRQ_TARGET_POS |
3720 		       IMR_TFH_SRV_DMA_CHNL0_CTRL_D2S_DMA_EN_POS |
3721 		       IMR_TFH_SRV_DMA_CHNL0_CTRL_D2S_RS_MSK);
3722 }
3723 
3724 int iwl_trans_pcie_copy_imr(struct iwl_trans *trans,
3725 			    u32 dst_addr, u64 src_addr, u32 byte_cnt)
3726 {
3727 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3728 	int ret = -1;
3729 
3730 	trans_pcie->imr_status = IMR_D2S_REQUESTED;
3731 	iwl_trans_pcie_copy_imr_fh(trans, dst_addr, src_addr, byte_cnt);
3732 	ret = wait_event_timeout(trans_pcie->imr_waitq,
3733 				 trans_pcie->imr_status !=
3734 				 IMR_D2S_REQUESTED, 5 * HZ);
3735 	if (!ret || trans_pcie->imr_status == IMR_D2S_ERROR) {
3736 		IWL_ERR(trans, "Failed to copy IMR Memory chunk!\n");
3737 		iwl_trans_pcie_dump_regs(trans);
3738 		return -ETIMEDOUT;
3739 	}
3740 	trans_pcie->imr_status = IMR_D2S_IDLE;
3741 	return 0;
3742 }
3743