1e705c121SKalle Valo /******************************************************************************
2e705c121SKalle Valo  *
3e705c121SKalle Valo  * This file is provided under a dual BSD/GPLv2 license.  When using or
4e705c121SKalle Valo  * redistributing this file, you may do so under either license.
5e705c121SKalle Valo  *
6e705c121SKalle Valo  * GPL LICENSE SUMMARY
7e705c121SKalle Valo  *
8e705c121SKalle Valo  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
9afb84431SEmmanuel Grumbach  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
10771db3a1SHaim Dreyfuss  * Copyright(c) 2007 - 2015, 2018 - 2020 Intel Corporation
11e705c121SKalle Valo  *
12e705c121SKalle Valo  * This program is free software; you can redistribute it and/or modify
13e705c121SKalle Valo  * it under the terms of version 2 of the GNU General Public License as
14e705c121SKalle Valo  * published by the Free Software Foundation.
15e705c121SKalle Valo  *
16e705c121SKalle Valo  * This program is distributed in the hope that it will be useful, but
17e705c121SKalle Valo  * WITHOUT ANY WARRANTY; without even the implied warranty of
18e705c121SKalle Valo  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
19e705c121SKalle Valo  * General Public License for more details.
20e705c121SKalle Valo  *
21e705c121SKalle Valo  * The full GNU General Public License is included in this distribution
22e705c121SKalle Valo  * in the file called COPYING.
23e705c121SKalle Valo  *
24e705c121SKalle Valo  * Contact Information:
25cb2f8277SEmmanuel Grumbach  *  Intel Linux Wireless <linuxwifi@intel.com>
26e705c121SKalle Valo  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
27e705c121SKalle Valo  *
28e705c121SKalle Valo  * BSD LICENSE
29e705c121SKalle Valo  *
30e705c121SKalle Valo  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
31afb84431SEmmanuel Grumbach  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
32771db3a1SHaim Dreyfuss  * Copyright(c) 2007 - 2015, 2018 - 2020 Intel Corporation
33e705c121SKalle Valo  * All rights reserved.
34e705c121SKalle Valo  *
35e705c121SKalle Valo  * Redistribution and use in source and binary forms, with or without
36e705c121SKalle Valo  * modification, are permitted provided that the following conditions
37e705c121SKalle Valo  * are met:
38e705c121SKalle Valo  *
39e705c121SKalle Valo  *  * Redistributions of source code must retain the above copyright
40e705c121SKalle Valo  *    notice, this list of conditions and the following disclaimer.
41e705c121SKalle Valo  *  * Redistributions in binary form must reproduce the above copyright
42e705c121SKalle Valo  *    notice, this list of conditions and the following disclaimer in
43e705c121SKalle Valo  *    the documentation and/or other materials provided with the
44e705c121SKalle Valo  *    distribution.
45e705c121SKalle Valo  *  * Neither the name Intel Corporation nor the names of its
46e705c121SKalle Valo  *    contributors may be used to endorse or promote products derived
47e705c121SKalle Valo  *    from this software without specific prior written permission.
48e705c121SKalle Valo  *
49e705c121SKalle Valo  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
50e705c121SKalle Valo  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
51e705c121SKalle Valo  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
52e705c121SKalle Valo  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
53e705c121SKalle Valo  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
54e705c121SKalle Valo  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
55e705c121SKalle Valo  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
56e705c121SKalle Valo  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
57e705c121SKalle Valo  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
58e705c121SKalle Valo  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
59e705c121SKalle Valo  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
60e705c121SKalle Valo  *
61e705c121SKalle Valo  *****************************************************************************/
62e705c121SKalle Valo #include <linux/pci.h>
63e705c121SKalle Valo #include <linux/interrupt.h>
64e705c121SKalle Valo #include <linux/debugfs.h>
65e705c121SKalle Valo #include <linux/sched.h>
66e705c121SKalle Valo #include <linux/bitops.h>
67e705c121SKalle Valo #include <linux/gfp.h>
68e705c121SKalle Valo #include <linux/vmalloc.h>
6949564a80SLuca Coelho #include <linux/module.h>
70f7805b33SLior Cohen #include <linux/wait.h>
71df67a1beSJohannes Berg #include <linux/seq_file.h>
72e705c121SKalle Valo 
73e705c121SKalle Valo #include "iwl-drv.h"
74e705c121SKalle Valo #include "iwl-trans.h"
75e705c121SKalle Valo #include "iwl-csr.h"
76e705c121SKalle Valo #include "iwl-prph.h"
77e705c121SKalle Valo #include "iwl-scd.h"
78e705c121SKalle Valo #include "iwl-agn-hw.h"
79d962f9b1SJohannes Berg #include "fw/error-dump.h"
80520f03eaSShahar S Matityahu #include "fw/dbg.h"
81a89c72ffSJohannes Berg #include "fw/api/tx.h"
82e705c121SKalle Valo #include "internal.h"
83e705c121SKalle Valo #include "iwl-fh.h"
84e705c121SKalle Valo 
85e705c121SKalle Valo /* extended range in FW SRAM */
86e705c121SKalle Valo #define IWL_FW_MEM_EXTENDED_START	0x40000
87e705c121SKalle Valo #define IWL_FW_MEM_EXTENDED_END		0x57FFF
88e705c121SKalle Valo 
894290eaadSJohannes Berg void iwl_trans_pcie_dump_regs(struct iwl_trans *trans)
90a6d24fadSRajat Jain {
91c4d3f2eeSLuca Coelho #define PCI_DUMP_SIZE		352
92c4d3f2eeSLuca Coelho #define PCI_MEM_DUMP_SIZE	64
93c4d3f2eeSLuca Coelho #define PCI_PARENT_DUMP_SIZE	524
94a6d24fadSRajat Jain #define PREFIX_LEN		32
95a6d24fadSRajat Jain 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
96a6d24fadSRajat Jain 	struct pci_dev *pdev = trans_pcie->pci_dev;
97a6d24fadSRajat Jain 	u32 i, pos, alloc_size, *ptr, *buf;
98a6d24fadSRajat Jain 	char *prefix;
99a6d24fadSRajat Jain 
100a6d24fadSRajat Jain 	if (trans_pcie->pcie_dbg_dumped_once)
101a6d24fadSRajat Jain 		return;
102a6d24fadSRajat Jain 
103a6d24fadSRajat Jain 	/* Should be a multiple of 4 */
104a6d24fadSRajat Jain 	BUILD_BUG_ON(PCI_DUMP_SIZE > 4096 || PCI_DUMP_SIZE & 0x3);
105c4d3f2eeSLuca Coelho 	BUILD_BUG_ON(PCI_MEM_DUMP_SIZE > 4096 || PCI_MEM_DUMP_SIZE & 0x3);
106c4d3f2eeSLuca Coelho 	BUILD_BUG_ON(PCI_PARENT_DUMP_SIZE > 4096 || PCI_PARENT_DUMP_SIZE & 0x3);
107c4d3f2eeSLuca Coelho 
108a6d24fadSRajat Jain 	/* Alloc a max size buffer */
109a6d24fadSRajat Jain 	alloc_size = PCI_ERR_ROOT_ERR_SRC +  4 + PREFIX_LEN;
110c4d3f2eeSLuca Coelho 	alloc_size = max_t(u32, alloc_size, PCI_DUMP_SIZE + PREFIX_LEN);
111c4d3f2eeSLuca Coelho 	alloc_size = max_t(u32, alloc_size, PCI_MEM_DUMP_SIZE + PREFIX_LEN);
112c4d3f2eeSLuca Coelho 	alloc_size = max_t(u32, alloc_size, PCI_PARENT_DUMP_SIZE + PREFIX_LEN);
113c4d3f2eeSLuca Coelho 
114a6d24fadSRajat Jain 	buf = kmalloc(alloc_size, GFP_ATOMIC);
115a6d24fadSRajat Jain 	if (!buf)
116a6d24fadSRajat Jain 		return;
117a6d24fadSRajat Jain 	prefix = (char *)buf + alloc_size - PREFIX_LEN;
118a6d24fadSRajat Jain 
119a6d24fadSRajat Jain 	IWL_ERR(trans, "iwlwifi transaction failed, dumping registers\n");
120a6d24fadSRajat Jain 
121a6d24fadSRajat Jain 	/* Print wifi device registers */
122a6d24fadSRajat Jain 	sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
123a6d24fadSRajat Jain 	IWL_ERR(trans, "iwlwifi device config registers:\n");
124a6d24fadSRajat Jain 	for (i = 0, ptr = buf; i < PCI_DUMP_SIZE; i += 4, ptr++)
125a6d24fadSRajat Jain 		if (pci_read_config_dword(pdev, i, ptr))
126a6d24fadSRajat Jain 			goto err_read;
127a6d24fadSRajat Jain 	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
128a6d24fadSRajat Jain 
129a6d24fadSRajat Jain 	IWL_ERR(trans, "iwlwifi device memory mapped registers:\n");
130c4d3f2eeSLuca Coelho 	for (i = 0, ptr = buf; i < PCI_MEM_DUMP_SIZE; i += 4, ptr++)
131a6d24fadSRajat Jain 		*ptr = iwl_read32(trans, i);
132a6d24fadSRajat Jain 	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
133a6d24fadSRajat Jain 
134a6d24fadSRajat Jain 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
135a6d24fadSRajat Jain 	if (pos) {
136a6d24fadSRajat Jain 		IWL_ERR(trans, "iwlwifi device AER capability structure:\n");
137a6d24fadSRajat Jain 		for (i = 0, ptr = buf; i < PCI_ERR_ROOT_COMMAND; i += 4, ptr++)
138a6d24fadSRajat Jain 			if (pci_read_config_dword(pdev, pos + i, ptr))
139a6d24fadSRajat Jain 				goto err_read;
140a6d24fadSRajat Jain 		print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET,
141a6d24fadSRajat Jain 			       32, 4, buf, i, 0);
142a6d24fadSRajat Jain 	}
143a6d24fadSRajat Jain 
144a6d24fadSRajat Jain 	/* Print parent device registers next */
145a6d24fadSRajat Jain 	if (!pdev->bus->self)
146a6d24fadSRajat Jain 		goto out;
147a6d24fadSRajat Jain 
148a6d24fadSRajat Jain 	pdev = pdev->bus->self;
149a6d24fadSRajat Jain 	sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
150a6d24fadSRajat Jain 
151a6d24fadSRajat Jain 	IWL_ERR(trans, "iwlwifi parent port (%s) config registers:\n",
152a6d24fadSRajat Jain 		pci_name(pdev));
153c4d3f2eeSLuca Coelho 	for (i = 0, ptr = buf; i < PCI_PARENT_DUMP_SIZE; i += 4, ptr++)
154a6d24fadSRajat Jain 		if (pci_read_config_dword(pdev, i, ptr))
155a6d24fadSRajat Jain 			goto err_read;
156a6d24fadSRajat Jain 	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
157a6d24fadSRajat Jain 
158a6d24fadSRajat Jain 	/* Print root port AER registers */
159a6d24fadSRajat Jain 	pos = 0;
160a6d24fadSRajat Jain 	pdev = pcie_find_root_port(pdev);
161a6d24fadSRajat Jain 	if (pdev)
162a6d24fadSRajat Jain 		pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
163a6d24fadSRajat Jain 	if (pos) {
164a6d24fadSRajat Jain 		IWL_ERR(trans, "iwlwifi root port (%s) AER cap structure:\n",
165a6d24fadSRajat Jain 			pci_name(pdev));
166a6d24fadSRajat Jain 		sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
167a6d24fadSRajat Jain 		for (i = 0, ptr = buf; i <= PCI_ERR_ROOT_ERR_SRC; i += 4, ptr++)
168a6d24fadSRajat Jain 			if (pci_read_config_dword(pdev, pos + i, ptr))
169a6d24fadSRajat Jain 				goto err_read;
170a6d24fadSRajat Jain 		print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32,
171a6d24fadSRajat Jain 			       4, buf, i, 0);
172a6d24fadSRajat Jain 	}
173f3402d6dSSara Sharon 	goto out;
174a6d24fadSRajat Jain 
175a6d24fadSRajat Jain err_read:
176a6d24fadSRajat Jain 	print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
177a6d24fadSRajat Jain 	IWL_ERR(trans, "Read failed at 0x%X\n", i);
178a6d24fadSRajat Jain out:
179a6d24fadSRajat Jain 	trans_pcie->pcie_dbg_dumped_once = 1;
180a6d24fadSRajat Jain 	kfree(buf);
181a6d24fadSRajat Jain }
182a6d24fadSRajat Jain 
183870c2a11SGolan Ben Ami static void iwl_trans_pcie_sw_reset(struct iwl_trans *trans)
184870c2a11SGolan Ben Ami {
185870c2a11SGolan Ben Ami 	/* Reset entire device - do controller reset (results in SHRD_HW_RST) */
1866dece0e9SLuca Coelho 	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
187870c2a11SGolan Ben Ami 	usleep_range(5000, 6000);
188870c2a11SGolan Ben Ami }
189870c2a11SGolan Ben Ami 
190e705c121SKalle Valo static void iwl_pcie_free_fw_monitor(struct iwl_trans *trans)
191e705c121SKalle Valo {
19269f0e505SShahar S Matityahu 	struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
193e705c121SKalle Valo 
19469f0e505SShahar S Matityahu 	if (!fw_mon->size)
19569f0e505SShahar S Matityahu 		return;
19669f0e505SShahar S Matityahu 
19769f0e505SShahar S Matityahu 	dma_free_coherent(trans->dev, fw_mon->size, fw_mon->block,
19869f0e505SShahar S Matityahu 			  fw_mon->physical);
19969f0e505SShahar S Matityahu 
20069f0e505SShahar S Matityahu 	fw_mon->block = NULL;
20169f0e505SShahar S Matityahu 	fw_mon->physical = 0;
20269f0e505SShahar S Matityahu 	fw_mon->size = 0;
203e705c121SKalle Valo }
204e705c121SKalle Valo 
20588964b2eSSara Sharon static void iwl_pcie_alloc_fw_monitor_block(struct iwl_trans *trans,
20688964b2eSSara Sharon 					    u8 max_power, u8 min_power)
207e705c121SKalle Valo {
20869f0e505SShahar S Matityahu 	struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
20969f0e505SShahar S Matityahu 	void *block = NULL;
21069f0e505SShahar S Matityahu 	dma_addr_t physical = 0;
211e705c121SKalle Valo 	u32 size = 0;
212e705c121SKalle Valo 	u8 power;
213e705c121SKalle Valo 
21469f0e505SShahar S Matityahu 	if (fw_mon->size)
21569f0e505SShahar S Matityahu 		return;
21669f0e505SShahar S Matityahu 
21788964b2eSSara Sharon 	for (power = max_power; power >= min_power; power--) {
218e705c121SKalle Valo 		size = BIT(power);
21969f0e505SShahar S Matityahu 		block = dma_alloc_coherent(trans->dev, size, &physical,
2202d46f7afSChristoph Hellwig 					   GFP_KERNEL | __GFP_NOWARN);
22169f0e505SShahar S Matityahu 		if (!block)
222e705c121SKalle Valo 			continue;
223e705c121SKalle Valo 
224e705c121SKalle Valo 		IWL_INFO(trans,
225c5f97542SShahar S Matityahu 			 "Allocated 0x%08x bytes for firmware monitor.\n",
226c5f97542SShahar S Matityahu 			 size);
227e705c121SKalle Valo 		break;
228e705c121SKalle Valo 	}
229e705c121SKalle Valo 
23069f0e505SShahar S Matityahu 	if (WARN_ON_ONCE(!block))
231e705c121SKalle Valo 		return;
232e705c121SKalle Valo 
233e705c121SKalle Valo 	if (power != max_power)
234e705c121SKalle Valo 		IWL_ERR(trans,
235e705c121SKalle Valo 			"Sorry - debug buffer is only %luK while you requested %luK\n",
236e705c121SKalle Valo 			(unsigned long)BIT(power - 10),
237e705c121SKalle Valo 			(unsigned long)BIT(max_power - 10));
238e705c121SKalle Valo 
23969f0e505SShahar S Matityahu 	fw_mon->block = block;
24069f0e505SShahar S Matityahu 	fw_mon->physical = physical;
24169f0e505SShahar S Matityahu 	fw_mon->size = size;
24288964b2eSSara Sharon }
24388964b2eSSara Sharon 
24488964b2eSSara Sharon void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power)
24588964b2eSSara Sharon {
24688964b2eSSara Sharon 	if (!max_power) {
24788964b2eSSara Sharon 		/* default max_power is maximum */
24888964b2eSSara Sharon 		max_power = 26;
24988964b2eSSara Sharon 	} else {
25088964b2eSSara Sharon 		max_power += 11;
25188964b2eSSara Sharon 	}
25288964b2eSSara Sharon 
25388964b2eSSara Sharon 	if (WARN(max_power > 26,
25488964b2eSSara Sharon 		 "External buffer size for monitor is too big %d, check the FW TLV\n",
25588964b2eSSara Sharon 		 max_power))
25688964b2eSSara Sharon 		return;
25788964b2eSSara Sharon 
25869f0e505SShahar S Matityahu 	if (trans->dbg.fw_mon.size)
25988964b2eSSara Sharon 		return;
26088964b2eSSara Sharon 
26188964b2eSSara Sharon 	iwl_pcie_alloc_fw_monitor_block(trans, max_power, 11);
262e705c121SKalle Valo }
263e705c121SKalle Valo 
264e705c121SKalle Valo static u32 iwl_trans_pcie_read_shr(struct iwl_trans *trans, u32 reg)
265e705c121SKalle Valo {
266e705c121SKalle Valo 	iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
267e705c121SKalle Valo 		    ((reg & 0x0000ffff) | (2 << 28)));
268e705c121SKalle Valo 	return iwl_read32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG);
269e705c121SKalle Valo }
270e705c121SKalle Valo 
271e705c121SKalle Valo static void iwl_trans_pcie_write_shr(struct iwl_trans *trans, u32 reg, u32 val)
272e705c121SKalle Valo {
273e705c121SKalle Valo 	iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG, val);
274e705c121SKalle Valo 	iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
275e705c121SKalle Valo 		    ((reg & 0x0000ffff) | (3 << 28)));
276e705c121SKalle Valo }
277e705c121SKalle Valo 
278e705c121SKalle Valo static void iwl_pcie_set_pwr(struct iwl_trans *trans, bool vaux)
279e705c121SKalle Valo {
280e705c121SKalle Valo 	if (trans->cfg->apmg_not_supported)
281e705c121SKalle Valo 		return;
282e705c121SKalle Valo 
283e705c121SKalle Valo 	if (vaux && pci_pme_capable(to_pci_dev(trans->dev), PCI_D3cold))
284e705c121SKalle Valo 		iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
285e705c121SKalle Valo 				       APMG_PS_CTRL_VAL_PWR_SRC_VAUX,
286e705c121SKalle Valo 				       ~APMG_PS_CTRL_MSK_PWR_SRC);
287e705c121SKalle Valo 	else
288e705c121SKalle Valo 		iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
289e705c121SKalle Valo 				       APMG_PS_CTRL_VAL_PWR_SRC_VMAIN,
290e705c121SKalle Valo 				       ~APMG_PS_CTRL_MSK_PWR_SRC);
291e705c121SKalle Valo }
292e705c121SKalle Valo 
293e705c121SKalle Valo /* PCI registers */
294e705c121SKalle Valo #define PCI_CFG_RETRY_TIMEOUT	0x041
295e705c121SKalle Valo 
296eda50cdeSSara Sharon void iwl_pcie_apm_config(struct iwl_trans *trans)
297e705c121SKalle Valo {
298e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
299e705c121SKalle Valo 	u16 lctl;
300e705c121SKalle Valo 	u16 cap;
301e705c121SKalle Valo 
302e705c121SKalle Valo 	/*
303cc894b85SLuca Coelho 	 * L0S states have been found to be unstable with our devices
304cc894b85SLuca Coelho 	 * and in newer hardware they are not officially supported at
305cc894b85SLuca Coelho 	 * all, so we must always set the L0S_DISABLED bit.
306e705c121SKalle Valo 	 */
3073d1b28fdSLuca Coelho 	iwl_set_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_DISABLED);
308cc894b85SLuca Coelho 
309cc894b85SLuca Coelho 	pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_LNKCTL, &lctl);
310e705c121SKalle Valo 	trans->pm_support = !(lctl & PCI_EXP_LNKCTL_ASPM_L0S);
311e705c121SKalle Valo 
312e705c121SKalle Valo 	pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_DEVCTL2, &cap);
313e705c121SKalle Valo 	trans->ltr_enabled = cap & PCI_EXP_DEVCTL2_LTR_EN;
314d74a61fcSLuca Coelho 	IWL_DEBUG_POWER(trans, "L1 %sabled - LTR %sabled\n",
315e705c121SKalle Valo 			(lctl & PCI_EXP_LNKCTL_ASPM_L1) ? "En" : "Dis",
316e705c121SKalle Valo 			trans->ltr_enabled ? "En" : "Dis");
317e705c121SKalle Valo }
318e705c121SKalle Valo 
319e705c121SKalle Valo /*
320e705c121SKalle Valo  * Start up NIC's basic functionality after it has been reset
321e705c121SKalle Valo  * (e.g. after platform boot, or shutdown via iwl_pcie_apm_stop())
322e705c121SKalle Valo  * NOTE:  This does not load uCode nor start the embedded processor
323e705c121SKalle Valo  */
324e705c121SKalle Valo static int iwl_pcie_apm_init(struct iwl_trans *trans)
325e705c121SKalle Valo {
32652b6e168SEmmanuel Grumbach 	int ret;
32752b6e168SEmmanuel Grumbach 
328e705c121SKalle Valo 	IWL_DEBUG_INFO(trans, "Init card's basic functions\n");
329e705c121SKalle Valo 
330e705c121SKalle Valo 	/*
331e705c121SKalle Valo 	 * Use "set_bit" below rather than "write", to preserve any hardware
332e705c121SKalle Valo 	 * bits already set by default after reset.
333e705c121SKalle Valo 	 */
334e705c121SKalle Valo 
335e705c121SKalle Valo 	/* Disable L0S exit timer (platform NMI Work/Around) */
336286ca8ebSLuca Coelho 	if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000)
337e705c121SKalle Valo 		iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
338e705c121SKalle Valo 			    CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER);
339e705c121SKalle Valo 
340e705c121SKalle Valo 	/*
341e705c121SKalle Valo 	 * Disable L0s without affecting L1;
342e705c121SKalle Valo 	 *  don't wait for ICH L0s (ICH bug W/A)
343e705c121SKalle Valo 	 */
344e705c121SKalle Valo 	iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
345e705c121SKalle Valo 		    CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX);
346e705c121SKalle Valo 
347e705c121SKalle Valo 	/* Set FH wait threshold to maximum (HW error during stress W/A) */
348e705c121SKalle Valo 	iwl_set_bit(trans, CSR_DBG_HPET_MEM_REG, CSR_DBG_HPET_MEM_REG_VAL);
349e705c121SKalle Valo 
350e705c121SKalle Valo 	/*
351e705c121SKalle Valo 	 * Enable HAP INTA (interrupt from management bus) to
352e705c121SKalle Valo 	 * wake device's PCI Express link L1a -> L0s
353e705c121SKalle Valo 	 */
354e705c121SKalle Valo 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
355e705c121SKalle Valo 		    CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
356e705c121SKalle Valo 
357e705c121SKalle Valo 	iwl_pcie_apm_config(trans);
358e705c121SKalle Valo 
359e705c121SKalle Valo 	/* Configure analog phase-lock-loop before activating to D0A */
360286ca8ebSLuca Coelho 	if (trans->trans_cfg->base_params->pll_cfg)
36177d76931SJohannes Berg 		iwl_set_bit(trans, CSR_ANA_PLL_CFG, CSR50_ANA_PLL_CFG_VAL);
362e705c121SKalle Valo 
3637d34a7d7SLuca Coelho 	ret = iwl_finish_nic_init(trans, trans->trans_cfg);
364c96b5eecSJohannes Berg 	if (ret)
36552b6e168SEmmanuel Grumbach 		return ret;
366e705c121SKalle Valo 
367e705c121SKalle Valo 	if (trans->cfg->host_interrupt_operation_mode) {
368e705c121SKalle Valo 		/*
369e705c121SKalle Valo 		 * This is a bit of an abuse - This is needed for 7260 / 3160
370e705c121SKalle Valo 		 * only check host_interrupt_operation_mode even if this is
371e705c121SKalle Valo 		 * not related to host_interrupt_operation_mode.
372e705c121SKalle Valo 		 *
373e705c121SKalle Valo 		 * Enable the oscillator to count wake up time for L1 exit. This
374e705c121SKalle Valo 		 * consumes slightly more power (100uA) - but allows to be sure
375e705c121SKalle Valo 		 * that we wake up from L1 on time.
376e705c121SKalle Valo 		 *
377e705c121SKalle Valo 		 * This looks weird: read twice the same register, discard the
378e705c121SKalle Valo 		 * value, set a bit, and yet again, read that same register
379e705c121SKalle Valo 		 * just to discard the value. But that's the way the hardware
380e705c121SKalle Valo 		 * seems to like it.
381e705c121SKalle Valo 		 */
382e705c121SKalle Valo 		iwl_read_prph(trans, OSC_CLK);
383e705c121SKalle Valo 		iwl_read_prph(trans, OSC_CLK);
384e705c121SKalle Valo 		iwl_set_bits_prph(trans, OSC_CLK, OSC_CLK_FORCE_CONTROL);
385e705c121SKalle Valo 		iwl_read_prph(trans, OSC_CLK);
386e705c121SKalle Valo 		iwl_read_prph(trans, OSC_CLK);
387e705c121SKalle Valo 	}
388e705c121SKalle Valo 
389e705c121SKalle Valo 	/*
390e705c121SKalle Valo 	 * Enable DMA clock and wait for it to stabilize.
391e705c121SKalle Valo 	 *
392e705c121SKalle Valo 	 * Write to "CLK_EN_REG"; "1" bits enable clocks, while "0"
393e705c121SKalle Valo 	 * bits do not disable clocks.  This preserves any hardware
394e705c121SKalle Valo 	 * bits already set by default in "CLK_CTRL_REG" after reset.
395e705c121SKalle Valo 	 */
396e705c121SKalle Valo 	if (!trans->cfg->apmg_not_supported) {
397e705c121SKalle Valo 		iwl_write_prph(trans, APMG_CLK_EN_REG,
398e705c121SKalle Valo 			       APMG_CLK_VAL_DMA_CLK_RQT);
399e705c121SKalle Valo 		udelay(20);
400e705c121SKalle Valo 
401e705c121SKalle Valo 		/* Disable L1-Active */
402e705c121SKalle Valo 		iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
403e705c121SKalle Valo 				  APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
404e705c121SKalle Valo 
405e705c121SKalle Valo 		/* Clear the interrupt in APMG if the NIC is in RFKILL */
406e705c121SKalle Valo 		iwl_write_prph(trans, APMG_RTC_INT_STT_REG,
407e705c121SKalle Valo 			       APMG_RTC_INT_STT_RFKILL);
408e705c121SKalle Valo 	}
409e705c121SKalle Valo 
410e705c121SKalle Valo 	set_bit(STATUS_DEVICE_ENABLED, &trans->status);
411e705c121SKalle Valo 
41252b6e168SEmmanuel Grumbach 	return 0;
413e705c121SKalle Valo }
414e705c121SKalle Valo 
415e705c121SKalle Valo /*
416e705c121SKalle Valo  * Enable LP XTAL to avoid HW bug where device may consume much power if
417e705c121SKalle Valo  * FW is not loaded after device reset. LP XTAL is disabled by default
418e705c121SKalle Valo  * after device HW reset. Do it only if XTAL is fed by internal source.
419e705c121SKalle Valo  * Configure device's "persistence" mode to avoid resetting XTAL again when
420e705c121SKalle Valo  * SHRD_HW_RST occurs in S3.
421e705c121SKalle Valo  */
422e705c121SKalle Valo static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
423e705c121SKalle Valo {
424e705c121SKalle Valo 	int ret;
425e705c121SKalle Valo 	u32 apmg_gp1_reg;
426e705c121SKalle Valo 	u32 apmg_xtal_cfg_reg;
427e705c121SKalle Valo 	u32 dl_cfg_reg;
428e705c121SKalle Valo 
429e705c121SKalle Valo 	/* Force XTAL ON */
430e705c121SKalle Valo 	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
431e705c121SKalle Valo 				 CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
432e705c121SKalle Valo 
433870c2a11SGolan Ben Ami 	iwl_trans_pcie_sw_reset(trans);
434e705c121SKalle Valo 
4357d34a7d7SLuca Coelho 	ret = iwl_finish_nic_init(trans, trans->trans_cfg);
436c96b5eecSJohannes Berg 	if (WARN_ON(ret)) {
437e705c121SKalle Valo 		/* Release XTAL ON request */
438e705c121SKalle Valo 		__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
439e705c121SKalle Valo 					   CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
440e705c121SKalle Valo 		return;
441e705c121SKalle Valo 	}
442e705c121SKalle Valo 
443e705c121SKalle Valo 	/*
444e705c121SKalle Valo 	 * Clear "disable persistence" to avoid LP XTAL resetting when
445e705c121SKalle Valo 	 * SHRD_HW_RST is applied in S3.
446e705c121SKalle Valo 	 */
447e705c121SKalle Valo 	iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
448e705c121SKalle Valo 				    APMG_PCIDEV_STT_VAL_PERSIST_DIS);
449e705c121SKalle Valo 
450e705c121SKalle Valo 	/*
451e705c121SKalle Valo 	 * Force APMG XTAL to be active to prevent its disabling by HW
452e705c121SKalle Valo 	 * caused by APMG idle state.
453e705c121SKalle Valo 	 */
454e705c121SKalle Valo 	apmg_xtal_cfg_reg = iwl_trans_pcie_read_shr(trans,
455e705c121SKalle Valo 						    SHR_APMG_XTAL_CFG_REG);
456e705c121SKalle Valo 	iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
457e705c121SKalle Valo 				 apmg_xtal_cfg_reg |
458e705c121SKalle Valo 				 SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
459e705c121SKalle Valo 
460870c2a11SGolan Ben Ami 	iwl_trans_pcie_sw_reset(trans);
461e705c121SKalle Valo 
462e705c121SKalle Valo 	/* Enable LP XTAL by indirect access through CSR */
463e705c121SKalle Valo 	apmg_gp1_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_GP1_REG);
464e705c121SKalle Valo 	iwl_trans_pcie_write_shr(trans, SHR_APMG_GP1_REG, apmg_gp1_reg |
465e705c121SKalle Valo 				 SHR_APMG_GP1_WF_XTAL_LP_EN |
466e705c121SKalle Valo 				 SHR_APMG_GP1_CHICKEN_BIT_SELECT);
467e705c121SKalle Valo 
468e705c121SKalle Valo 	/* Clear delay line clock power up */
469e705c121SKalle Valo 	dl_cfg_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_DL_CFG_REG);
470e705c121SKalle Valo 	iwl_trans_pcie_write_shr(trans, SHR_APMG_DL_CFG_REG, dl_cfg_reg &
471e705c121SKalle Valo 				 ~SHR_APMG_DL_CFG_DL_CLOCK_POWER_UP);
472e705c121SKalle Valo 
473e705c121SKalle Valo 	/*
474e705c121SKalle Valo 	 * Enable persistence mode to avoid LP XTAL resetting when
475e705c121SKalle Valo 	 * SHRD_HW_RST is applied in S3.
476e705c121SKalle Valo 	 */
477e705c121SKalle Valo 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
478e705c121SKalle Valo 		    CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
479e705c121SKalle Valo 
480e705c121SKalle Valo 	/*
481e705c121SKalle Valo 	 * Clear "initialization complete" bit to move adapter from
482e705c121SKalle Valo 	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
483e705c121SKalle Valo 	 */
4846dece0e9SLuca Coelho 	iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
485e705c121SKalle Valo 
486e705c121SKalle Valo 	/* Activates XTAL resources monitor */
487e705c121SKalle Valo 	__iwl_trans_pcie_set_bit(trans, CSR_MONITOR_CFG_REG,
488e705c121SKalle Valo 				 CSR_MONITOR_XTAL_RESOURCES);
489e705c121SKalle Valo 
490e705c121SKalle Valo 	/* Release XTAL ON request */
491e705c121SKalle Valo 	__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
492e705c121SKalle Valo 				   CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
493e705c121SKalle Valo 	udelay(10);
494e705c121SKalle Valo 
495e705c121SKalle Valo 	/* Release APMG XTAL */
496e705c121SKalle Valo 	iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
497e705c121SKalle Valo 				 apmg_xtal_cfg_reg &
498e705c121SKalle Valo 				 ~SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
499e705c121SKalle Valo }
500e705c121SKalle Valo 
501e8c8935eSJohannes Berg void iwl_pcie_apm_stop_master(struct iwl_trans *trans)
502e705c121SKalle Valo {
503e8c8935eSJohannes Berg 	int ret;
504e705c121SKalle Valo 
505e705c121SKalle Valo 	/* stop device's busmaster DMA activity */
5066dece0e9SLuca Coelho 	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_STOP_MASTER);
507e705c121SKalle Valo 
5086dece0e9SLuca Coelho 	ret = iwl_poll_bit(trans, CSR_RESET,
5096dece0e9SLuca Coelho 			   CSR_RESET_REG_FLAG_MASTER_DISABLED,
5106dece0e9SLuca Coelho 			   CSR_RESET_REG_FLAG_MASTER_DISABLED, 100);
511e705c121SKalle Valo 	if (ret < 0)
512e705c121SKalle Valo 		IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n");
513e705c121SKalle Valo 
514e705c121SKalle Valo 	IWL_DEBUG_INFO(trans, "stop master\n");
515e705c121SKalle Valo }
516e705c121SKalle Valo 
517e705c121SKalle Valo static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
518e705c121SKalle Valo {
519e705c121SKalle Valo 	IWL_DEBUG_INFO(trans, "Stop card, put in low power state\n");
520e705c121SKalle Valo 
521e705c121SKalle Valo 	if (op_mode_leave) {
522e705c121SKalle Valo 		if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
523e705c121SKalle Valo 			iwl_pcie_apm_init(trans);
524e705c121SKalle Valo 
525e705c121SKalle Valo 		/* inform ME that we are leaving */
526286ca8ebSLuca Coelho 		if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000)
527e705c121SKalle Valo 			iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
528e705c121SKalle Valo 					  APMG_PCIDEV_STT_VAL_WAKE_ME);
529286ca8ebSLuca Coelho 		else if (trans->trans_cfg->device_family >=
53079b6c8feSLuca Coelho 			 IWL_DEVICE_FAMILY_8000) {
531e705c121SKalle Valo 			iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
532e705c121SKalle Valo 				    CSR_RESET_LINK_PWR_MGMT_DISABLED);
533e705c121SKalle Valo 			iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
534e705c121SKalle Valo 				    CSR_HW_IF_CONFIG_REG_PREPARE |
535e705c121SKalle Valo 				    CSR_HW_IF_CONFIG_REG_ENABLE_PME);
536e705c121SKalle Valo 			mdelay(1);
537e705c121SKalle Valo 			iwl_clear_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
538e705c121SKalle Valo 				      CSR_RESET_LINK_PWR_MGMT_DISABLED);
539e705c121SKalle Valo 		}
540e705c121SKalle Valo 		mdelay(5);
541e705c121SKalle Valo 	}
542e705c121SKalle Valo 
543e705c121SKalle Valo 	clear_bit(STATUS_DEVICE_ENABLED, &trans->status);
544e705c121SKalle Valo 
545e705c121SKalle Valo 	/* Stop device's DMA activity */
546e705c121SKalle Valo 	iwl_pcie_apm_stop_master(trans);
547e705c121SKalle Valo 
548e705c121SKalle Valo 	if (trans->cfg->lp_xtal_workaround) {
549e705c121SKalle Valo 		iwl_pcie_apm_lp_xtal_enable(trans);
550e705c121SKalle Valo 		return;
551e705c121SKalle Valo 	}
552e705c121SKalle Valo 
553870c2a11SGolan Ben Ami 	iwl_trans_pcie_sw_reset(trans);
554e705c121SKalle Valo 
555e705c121SKalle Valo 	/*
556e705c121SKalle Valo 	 * Clear "initialization complete" bit to move adapter from
557e705c121SKalle Valo 	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
558e705c121SKalle Valo 	 */
5596dece0e9SLuca Coelho 	iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
560e705c121SKalle Valo }
561e705c121SKalle Valo 
562e705c121SKalle Valo static int iwl_pcie_nic_init(struct iwl_trans *trans)
563e705c121SKalle Valo {
564e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
56552b6e168SEmmanuel Grumbach 	int ret;
566e705c121SKalle Valo 
567e705c121SKalle Valo 	/* nic_init */
568e705c121SKalle Valo 	spin_lock(&trans_pcie->irq_lock);
56952b6e168SEmmanuel Grumbach 	ret = iwl_pcie_apm_init(trans);
570e705c121SKalle Valo 	spin_unlock(&trans_pcie->irq_lock);
571e705c121SKalle Valo 
57252b6e168SEmmanuel Grumbach 	if (ret)
57352b6e168SEmmanuel Grumbach 		return ret;
57452b6e168SEmmanuel Grumbach 
575e705c121SKalle Valo 	iwl_pcie_set_pwr(trans, false);
576e705c121SKalle Valo 
577e705c121SKalle Valo 	iwl_op_mode_nic_config(trans->op_mode);
578e705c121SKalle Valo 
579e705c121SKalle Valo 	/* Allocate the RX queue, or reset if it is already allocated */
580e705c121SKalle Valo 	iwl_pcie_rx_init(trans);
581e705c121SKalle Valo 
582e705c121SKalle Valo 	/* Allocate or reset and init all Tx and Command queues */
583e705c121SKalle Valo 	if (iwl_pcie_tx_init(trans))
584e705c121SKalle Valo 		return -ENOMEM;
585e705c121SKalle Valo 
586286ca8ebSLuca Coelho 	if (trans->trans_cfg->base_params->shadow_reg_enable) {
587e705c121SKalle Valo 		/* enable shadow regs in HW */
588e705c121SKalle Valo 		iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, 0x800FFFFF);
589e705c121SKalle Valo 		IWL_DEBUG_INFO(trans, "Enabling shadow registers in device\n");
590e705c121SKalle Valo 	}
591e705c121SKalle Valo 
592e705c121SKalle Valo 	return 0;
593e705c121SKalle Valo }
594e705c121SKalle Valo 
595e705c121SKalle Valo #define HW_READY_TIMEOUT (50)
596e705c121SKalle Valo 
597e705c121SKalle Valo /* Note: returns poll_bit return value, which is >= 0 if success */
598e705c121SKalle Valo static int iwl_pcie_set_hw_ready(struct iwl_trans *trans)
599e705c121SKalle Valo {
600e705c121SKalle Valo 	int ret;
601e705c121SKalle Valo 
602e705c121SKalle Valo 	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
603e705c121SKalle Valo 		    CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);
604e705c121SKalle Valo 
605e705c121SKalle Valo 	/* See if we got it */
606e705c121SKalle Valo 	ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
607e705c121SKalle Valo 			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
608e705c121SKalle Valo 			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
609e705c121SKalle Valo 			   HW_READY_TIMEOUT);
610e705c121SKalle Valo 
611e705c121SKalle Valo 	if (ret >= 0)
612e705c121SKalle Valo 		iwl_set_bit(trans, CSR_MBOX_SET_REG, CSR_MBOX_SET_REG_OS_ALIVE);
613e705c121SKalle Valo 
614e705c121SKalle Valo 	IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : "");
615e705c121SKalle Valo 	return ret;
616e705c121SKalle Valo }
617e705c121SKalle Valo 
618e705c121SKalle Valo /* Note: returns standard 0/-ERROR code */
619eda50cdeSSara Sharon int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
620e705c121SKalle Valo {
621e705c121SKalle Valo 	int ret;
622e705c121SKalle Valo 	int t = 0;
623e705c121SKalle Valo 	int iter;
624e705c121SKalle Valo 
625e705c121SKalle Valo 	IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n");
626e705c121SKalle Valo 
627e705c121SKalle Valo 	ret = iwl_pcie_set_hw_ready(trans);
628e705c121SKalle Valo 	/* If the card is ready, exit 0 */
629e705c121SKalle Valo 	if (ret >= 0)
630e705c121SKalle Valo 		return 0;
631e705c121SKalle Valo 
632e705c121SKalle Valo 	iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
633e705c121SKalle Valo 		    CSR_RESET_LINK_PWR_MGMT_DISABLED);
634192185d6SJohannes Berg 	usleep_range(1000, 2000);
635e705c121SKalle Valo 
636e705c121SKalle Valo 	for (iter = 0; iter < 10; iter++) {
637e705c121SKalle Valo 		/* If HW is not ready, prepare the conditions to check again */
638e705c121SKalle Valo 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
639e705c121SKalle Valo 			    CSR_HW_IF_CONFIG_REG_PREPARE);
640e705c121SKalle Valo 
641e705c121SKalle Valo 		do {
642e705c121SKalle Valo 			ret = iwl_pcie_set_hw_ready(trans);
643e705c121SKalle Valo 			if (ret >= 0)
644e705c121SKalle Valo 				return 0;
645e705c121SKalle Valo 
646e705c121SKalle Valo 			usleep_range(200, 1000);
647e705c121SKalle Valo 			t += 200;
648e705c121SKalle Valo 		} while (t < 150000);
649e705c121SKalle Valo 		msleep(25);
650e705c121SKalle Valo 	}
651e705c121SKalle Valo 
652e705c121SKalle Valo 	IWL_ERR(trans, "Couldn't prepare the card\n");
653e705c121SKalle Valo 
654e705c121SKalle Valo 	return ret;
655e705c121SKalle Valo }
656e705c121SKalle Valo 
657e705c121SKalle Valo /*
658e705c121SKalle Valo  * ucode
659e705c121SKalle Valo  */
660564cdce7SSara Sharon static void iwl_pcie_load_firmware_chunk_fh(struct iwl_trans *trans,
661564cdce7SSara Sharon 					    u32 dst_addr, dma_addr_t phy_addr,
662564cdce7SSara Sharon 					    u32 byte_cnt)
663e705c121SKalle Valo {
664bac842daSEmmanuel Grumbach 	iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
665e705c121SKalle Valo 		    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
666e705c121SKalle Valo 
667bac842daSEmmanuel Grumbach 	iwl_write32(trans, FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL),
668e705c121SKalle Valo 		    dst_addr);
669e705c121SKalle Valo 
670bac842daSEmmanuel Grumbach 	iwl_write32(trans, FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL),
671e705c121SKalle Valo 		    phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
672e705c121SKalle Valo 
673bac842daSEmmanuel Grumbach 	iwl_write32(trans, FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL),
674e705c121SKalle Valo 		    (iwl_get_dma_hi_addr(phy_addr)
675e705c121SKalle Valo 			<< FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
676e705c121SKalle Valo 
677bac842daSEmmanuel Grumbach 	iwl_write32(trans, FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL),
678bac842daSEmmanuel Grumbach 		    BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM) |
679bac842daSEmmanuel Grumbach 		    BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX) |
680e705c121SKalle Valo 		    FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
681e705c121SKalle Valo 
682bac842daSEmmanuel Grumbach 	iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
683e705c121SKalle Valo 		    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
684e705c121SKalle Valo 		    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
685e705c121SKalle Valo 		    FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
686564cdce7SSara Sharon }
687e705c121SKalle Valo 
688564cdce7SSara Sharon static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans,
689564cdce7SSara Sharon 					u32 dst_addr, dma_addr_t phy_addr,
690564cdce7SSara Sharon 					u32 byte_cnt)
691564cdce7SSara Sharon {
692564cdce7SSara Sharon 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
693564cdce7SSara Sharon 	unsigned long flags;
694564cdce7SSara Sharon 	int ret;
695564cdce7SSara Sharon 
696564cdce7SSara Sharon 	trans_pcie->ucode_write_complete = false;
697564cdce7SSara Sharon 
698564cdce7SSara Sharon 	if (!iwl_trans_grab_nic_access(trans, &flags))
699564cdce7SSara Sharon 		return -EIO;
700564cdce7SSara Sharon 
701564cdce7SSara Sharon 	iwl_pcie_load_firmware_chunk_fh(trans, dst_addr, phy_addr,
702564cdce7SSara Sharon 					byte_cnt);
703bac842daSEmmanuel Grumbach 	iwl_trans_release_nic_access(trans, &flags);
704bac842daSEmmanuel Grumbach 
705e705c121SKalle Valo 	ret = wait_event_timeout(trans_pcie->ucode_write_waitq,
706e705c121SKalle Valo 				 trans_pcie->ucode_write_complete, 5 * HZ);
707e705c121SKalle Valo 	if (!ret) {
708e705c121SKalle Valo 		IWL_ERR(trans, "Failed to load firmware chunk!\n");
709fb12777aSKirtika Ruchandani 		iwl_trans_pcie_dump_regs(trans);
710e705c121SKalle Valo 		return -ETIMEDOUT;
711e705c121SKalle Valo 	}
712e705c121SKalle Valo 
713e705c121SKalle Valo 	return 0;
714e705c121SKalle Valo }
715e705c121SKalle Valo 
716e705c121SKalle Valo static int iwl_pcie_load_section(struct iwl_trans *trans, u8 section_num,
717e705c121SKalle Valo 			    const struct fw_desc *section)
718e705c121SKalle Valo {
719e705c121SKalle Valo 	u8 *v_addr;
720e705c121SKalle Valo 	dma_addr_t p_addr;
721e705c121SKalle Valo 	u32 offset, chunk_sz = min_t(u32, FH_MEM_TB_MAX_LENGTH, section->len);
722e705c121SKalle Valo 	int ret = 0;
723e705c121SKalle Valo 
724e705c121SKalle Valo 	IWL_DEBUG_FW(trans, "[%d] uCode section being loaded...\n",
725e705c121SKalle Valo 		     section_num);
726e705c121SKalle Valo 
727e705c121SKalle Valo 	v_addr = dma_alloc_coherent(trans->dev, chunk_sz, &p_addr,
728e705c121SKalle Valo 				    GFP_KERNEL | __GFP_NOWARN);
729e705c121SKalle Valo 	if (!v_addr) {
730e705c121SKalle Valo 		IWL_DEBUG_INFO(trans, "Falling back to small chunks of DMA\n");
731e705c121SKalle Valo 		chunk_sz = PAGE_SIZE;
732e705c121SKalle Valo 		v_addr = dma_alloc_coherent(trans->dev, chunk_sz,
733e705c121SKalle Valo 					    &p_addr, GFP_KERNEL);
734e705c121SKalle Valo 		if (!v_addr)
735e705c121SKalle Valo 			return -ENOMEM;
736e705c121SKalle Valo 	}
737e705c121SKalle Valo 
738e705c121SKalle Valo 	for (offset = 0; offset < section->len; offset += chunk_sz) {
739e705c121SKalle Valo 		u32 copy_size, dst_addr;
740e705c121SKalle Valo 		bool extended_addr = false;
741e705c121SKalle Valo 
742e705c121SKalle Valo 		copy_size = min_t(u32, chunk_sz, section->len - offset);
743e705c121SKalle Valo 		dst_addr = section->offset + offset;
744e705c121SKalle Valo 
745e705c121SKalle Valo 		if (dst_addr >= IWL_FW_MEM_EXTENDED_START &&
746e705c121SKalle Valo 		    dst_addr <= IWL_FW_MEM_EXTENDED_END)
747e705c121SKalle Valo 			extended_addr = true;
748e705c121SKalle Valo 
749e705c121SKalle Valo 		if (extended_addr)
750e705c121SKalle Valo 			iwl_set_bits_prph(trans, LMPM_CHICK,
751e705c121SKalle Valo 					  LMPM_CHICK_EXTENDED_ADDR_SPACE);
752e705c121SKalle Valo 
753e705c121SKalle Valo 		memcpy(v_addr, (u8 *)section->data + offset, copy_size);
754e705c121SKalle Valo 		ret = iwl_pcie_load_firmware_chunk(trans, dst_addr, p_addr,
755e705c121SKalle Valo 						   copy_size);
756e705c121SKalle Valo 
757e705c121SKalle Valo 		if (extended_addr)
758e705c121SKalle Valo 			iwl_clear_bits_prph(trans, LMPM_CHICK,
759e705c121SKalle Valo 					    LMPM_CHICK_EXTENDED_ADDR_SPACE);
760e705c121SKalle Valo 
761e705c121SKalle Valo 		if (ret) {
762e705c121SKalle Valo 			IWL_ERR(trans,
763e705c121SKalle Valo 				"Could not load the [%d] uCode section\n",
764e705c121SKalle Valo 				section_num);
765e705c121SKalle Valo 			break;
766e705c121SKalle Valo 		}
767e705c121SKalle Valo 	}
768e705c121SKalle Valo 
769e705c121SKalle Valo 	dma_free_coherent(trans->dev, chunk_sz, v_addr, p_addr);
770e705c121SKalle Valo 	return ret;
771e705c121SKalle Valo }
772e705c121SKalle Valo 
773e705c121SKalle Valo static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans,
774e705c121SKalle Valo 					   const struct fw_img *image,
775e705c121SKalle Valo 					   int cpu,
776e705c121SKalle Valo 					   int *first_ucode_section)
777e705c121SKalle Valo {
778e705c121SKalle Valo 	int shift_param;
779e705c121SKalle Valo 	int i, ret = 0, sec_num = 0x1;
780e705c121SKalle Valo 	u32 val, last_read_idx = 0;
781e705c121SKalle Valo 
782e705c121SKalle Valo 	if (cpu == 1) {
783e705c121SKalle Valo 		shift_param = 0;
784e705c121SKalle Valo 		*first_ucode_section = 0;
785e705c121SKalle Valo 	} else {
786e705c121SKalle Valo 		shift_param = 16;
787e705c121SKalle Valo 		(*first_ucode_section)++;
788e705c121SKalle Valo 	}
789e705c121SKalle Valo 
790eef187a7SSara Sharon 	for (i = *first_ucode_section; i < image->num_sec; i++) {
791e705c121SKalle Valo 		last_read_idx = i;
792e705c121SKalle Valo 
793e705c121SKalle Valo 		/*
794e705c121SKalle Valo 		 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
795e705c121SKalle Valo 		 * CPU1 to CPU2.
796e705c121SKalle Valo 		 * PAGING_SEPARATOR_SECTION delimiter - separate between
797e705c121SKalle Valo 		 * CPU2 non paged to CPU2 paging sec.
798e705c121SKalle Valo 		 */
799e705c121SKalle Valo 		if (!image->sec[i].data ||
800e705c121SKalle Valo 		    image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
801e705c121SKalle Valo 		    image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
802e705c121SKalle Valo 			IWL_DEBUG_FW(trans,
803e705c121SKalle Valo 				     "Break since Data not valid or Empty section, sec = %d\n",
804e705c121SKalle Valo 				     i);
805e705c121SKalle Valo 			break;
806e705c121SKalle Valo 		}
807e705c121SKalle Valo 
808e705c121SKalle Valo 		ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
809e705c121SKalle Valo 		if (ret)
810e705c121SKalle Valo 			return ret;
811e705c121SKalle Valo 
812d6a2c5c7SSara Sharon 		/* Notify ucode of loaded section number and status */
813e705c121SKalle Valo 		val = iwl_read_direct32(trans, FH_UCODE_LOAD_STATUS);
814e705c121SKalle Valo 		val = val | (sec_num << shift_param);
815e705c121SKalle Valo 		iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, val);
816eda50cdeSSara Sharon 
817e705c121SKalle Valo 		sec_num = (sec_num << 1) | 0x1;
818e705c121SKalle Valo 	}
819e705c121SKalle Valo 
820e705c121SKalle Valo 	*first_ucode_section = last_read_idx;
821e705c121SKalle Valo 
8222aabdbdcSEmmanuel Grumbach 	iwl_enable_interrupts(trans);
8232aabdbdcSEmmanuel Grumbach 
824286ca8ebSLuca Coelho 	if (trans->trans_cfg->use_tfh) {
825e705c121SKalle Valo 		if (cpu == 1)
826d6a2c5c7SSara Sharon 			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
827d6a2c5c7SSara Sharon 				       0xFFFF);
828e705c121SKalle Valo 		else
829d6a2c5c7SSara Sharon 			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
830d6a2c5c7SSara Sharon 				       0xFFFFFFFF);
831d6a2c5c7SSara Sharon 	} else {
832d6a2c5c7SSara Sharon 		if (cpu == 1)
833d6a2c5c7SSara Sharon 			iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
834d6a2c5c7SSara Sharon 					   0xFFFF);
835d6a2c5c7SSara Sharon 		else
836d6a2c5c7SSara Sharon 			iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
837d6a2c5c7SSara Sharon 					   0xFFFFFFFF);
838d6a2c5c7SSara Sharon 	}
839e705c121SKalle Valo 
840e705c121SKalle Valo 	return 0;
841e705c121SKalle Valo }
842e705c121SKalle Valo 
843e705c121SKalle Valo static int iwl_pcie_load_cpu_sections(struct iwl_trans *trans,
844e705c121SKalle Valo 				      const struct fw_img *image,
845e705c121SKalle Valo 				      int cpu,
846e705c121SKalle Valo 				      int *first_ucode_section)
847e705c121SKalle Valo {
848e705c121SKalle Valo 	int i, ret = 0;
849e705c121SKalle Valo 	u32 last_read_idx = 0;
850e705c121SKalle Valo 
8513ce4a038SKirtika Ruchandani 	if (cpu == 1)
852e705c121SKalle Valo 		*first_ucode_section = 0;
8533ce4a038SKirtika Ruchandani 	else
854e705c121SKalle Valo 		(*first_ucode_section)++;
855e705c121SKalle Valo 
856eef187a7SSara Sharon 	for (i = *first_ucode_section; i < image->num_sec; i++) {
857e705c121SKalle Valo 		last_read_idx = i;
858e705c121SKalle Valo 
859e705c121SKalle Valo 		/*
860e705c121SKalle Valo 		 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
861e705c121SKalle Valo 		 * CPU1 to CPU2.
862e705c121SKalle Valo 		 * PAGING_SEPARATOR_SECTION delimiter - separate between
863e705c121SKalle Valo 		 * CPU2 non paged to CPU2 paging sec.
864e705c121SKalle Valo 		 */
865e705c121SKalle Valo 		if (!image->sec[i].data ||
866e705c121SKalle Valo 		    image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
867e705c121SKalle Valo 		    image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
868e705c121SKalle Valo 			IWL_DEBUG_FW(trans,
869e705c121SKalle Valo 				     "Break since Data not valid or Empty section, sec = %d\n",
870e705c121SKalle Valo 				     i);
871e705c121SKalle Valo 			break;
872e705c121SKalle Valo 		}
873e705c121SKalle Valo 
874e705c121SKalle Valo 		ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
875e705c121SKalle Valo 		if (ret)
876e705c121SKalle Valo 			return ret;
877e705c121SKalle Valo 	}
878e705c121SKalle Valo 
879e705c121SKalle Valo 	*first_ucode_section = last_read_idx;
880e705c121SKalle Valo 
881e705c121SKalle Valo 	return 0;
882e705c121SKalle Valo }
883e705c121SKalle Valo 
884593fae3eSShahar S Matityahu static void iwl_pcie_apply_destination_ini(struct iwl_trans *trans)
885593fae3eSShahar S Matityahu {
886593fae3eSShahar S Matityahu 	enum iwl_fw_ini_allocation_id alloc_id = IWL_FW_INI_ALLOCATION_ID_DBGC1;
887593fae3eSShahar S Matityahu 	struct iwl_fw_ini_allocation_tlv *fw_mon_cfg =
888593fae3eSShahar S Matityahu 		&trans->dbg.fw_mon_cfg[alloc_id];
889593fae3eSShahar S Matityahu 	struct iwl_dram_data *frag;
890593fae3eSShahar S Matityahu 
891593fae3eSShahar S Matityahu 	if (!iwl_trans_dbg_ini_valid(trans))
892593fae3eSShahar S Matityahu 		return;
893593fae3eSShahar S Matityahu 
894593fae3eSShahar S Matityahu 	if (le32_to_cpu(fw_mon_cfg->buf_location) ==
895593fae3eSShahar S Matityahu 	    IWL_FW_INI_LOCATION_SRAM_PATH) {
896593fae3eSShahar S Matityahu 		IWL_DEBUG_FW(trans, "WRT: Applying SMEM buffer destination\n");
897593fae3eSShahar S Matityahu 		/* set sram monitor by enabling bit 7 */
898593fae3eSShahar S Matityahu 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
899593fae3eSShahar S Matityahu 			    CSR_HW_IF_CONFIG_REG_BIT_MONITOR_SRAM);
900593fae3eSShahar S Matityahu 
901593fae3eSShahar S Matityahu 		return;
902593fae3eSShahar S Matityahu 	}
903593fae3eSShahar S Matityahu 
904593fae3eSShahar S Matityahu 	if (le32_to_cpu(fw_mon_cfg->buf_location) !=
905593fae3eSShahar S Matityahu 	    IWL_FW_INI_LOCATION_DRAM_PATH ||
906593fae3eSShahar S Matityahu 	    !trans->dbg.fw_mon_ini[alloc_id].num_frags)
907593fae3eSShahar S Matityahu 		return;
908593fae3eSShahar S Matityahu 
909593fae3eSShahar S Matityahu 	frag = &trans->dbg.fw_mon_ini[alloc_id].frags[0];
910593fae3eSShahar S Matityahu 
911593fae3eSShahar S Matityahu 	IWL_DEBUG_FW(trans, "WRT: Applying DRAM destination (alloc_id=%u)\n",
912593fae3eSShahar S Matityahu 		     alloc_id);
913593fae3eSShahar S Matityahu 
914593fae3eSShahar S Matityahu 	iwl_write_umac_prph(trans, MON_BUFF_BASE_ADDR_VER2,
915593fae3eSShahar S Matityahu 			    frag->physical >> MON_BUFF_SHIFT_VER2);
916593fae3eSShahar S Matityahu 	iwl_write_umac_prph(trans, MON_BUFF_END_ADDR_VER2,
917593fae3eSShahar S Matityahu 			    (frag->physical + frag->size - 256) >>
918593fae3eSShahar S Matityahu 			    MON_BUFF_SHIFT_VER2);
919593fae3eSShahar S Matityahu }
920593fae3eSShahar S Matityahu 
921c9be849dSLiad Kaufman void iwl_pcie_apply_destination(struct iwl_trans *trans)
922e705c121SKalle Valo {
92391c28b83SShahar S Matityahu 	const struct iwl_fw_dbg_dest_tlv_v1 *dest = trans->dbg.dest_tlv;
92469f0e505SShahar S Matityahu 	const struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
925e705c121SKalle Valo 	int i;
926e705c121SKalle Valo 
927a1af4c48SShahar S Matityahu 	if (iwl_trans_dbg_ini_valid(trans)) {
928593fae3eSShahar S Matityahu 		iwl_pcie_apply_destination_ini(trans);
9297a14c23dSSara Sharon 		return;
9307a14c23dSSara Sharon 	}
9317a14c23dSSara Sharon 
932e705c121SKalle Valo 	IWL_INFO(trans, "Applying debug destination %s\n",
933e705c121SKalle Valo 		 get_fw_dbg_mode_string(dest->monitor_mode));
934e705c121SKalle Valo 
935e705c121SKalle Valo 	if (dest->monitor_mode == EXTERNAL_MODE)
936e705c121SKalle Valo 		iwl_pcie_alloc_fw_monitor(trans, dest->size_power);
937e705c121SKalle Valo 	else
938e705c121SKalle Valo 		IWL_WARN(trans, "PCI should have external buffer debug\n");
939e705c121SKalle Valo 
94091c28b83SShahar S Matityahu 	for (i = 0; i < trans->dbg.n_dest_reg; i++) {
941e705c121SKalle Valo 		u32 addr = le32_to_cpu(dest->reg_ops[i].addr);
942e705c121SKalle Valo 		u32 val = le32_to_cpu(dest->reg_ops[i].val);
943e705c121SKalle Valo 
944e705c121SKalle Valo 		switch (dest->reg_ops[i].op) {
945e705c121SKalle Valo 		case CSR_ASSIGN:
946e705c121SKalle Valo 			iwl_write32(trans, addr, val);
947e705c121SKalle Valo 			break;
948e705c121SKalle Valo 		case CSR_SETBIT:
949e705c121SKalle Valo 			iwl_set_bit(trans, addr, BIT(val));
950e705c121SKalle Valo 			break;
951e705c121SKalle Valo 		case CSR_CLEARBIT:
952e705c121SKalle Valo 			iwl_clear_bit(trans, addr, BIT(val));
953e705c121SKalle Valo 			break;
954e705c121SKalle Valo 		case PRPH_ASSIGN:
955e705c121SKalle Valo 			iwl_write_prph(trans, addr, val);
956e705c121SKalle Valo 			break;
957e705c121SKalle Valo 		case PRPH_SETBIT:
958e705c121SKalle Valo 			iwl_set_bits_prph(trans, addr, BIT(val));
959e705c121SKalle Valo 			break;
960e705c121SKalle Valo 		case PRPH_CLEARBIT:
961e705c121SKalle Valo 			iwl_clear_bits_prph(trans, addr, BIT(val));
962e705c121SKalle Valo 			break;
963e705c121SKalle Valo 		case PRPH_BLOCKBIT:
964e705c121SKalle Valo 			if (iwl_read_prph(trans, addr) & BIT(val)) {
965e705c121SKalle Valo 				IWL_ERR(trans,
966e705c121SKalle Valo 					"BIT(%u) in address 0x%x is 1, stopping FW configuration\n",
967e705c121SKalle Valo 					val, addr);
968e705c121SKalle Valo 				goto monitor;
969e705c121SKalle Valo 			}
970e705c121SKalle Valo 			break;
971e705c121SKalle Valo 		default:
972e705c121SKalle Valo 			IWL_ERR(trans, "FW debug - unknown OP %d\n",
973e705c121SKalle Valo 				dest->reg_ops[i].op);
974e705c121SKalle Valo 			break;
975e705c121SKalle Valo 		}
976e705c121SKalle Valo 	}
977e705c121SKalle Valo 
978e705c121SKalle Valo monitor:
97969f0e505SShahar S Matityahu 	if (dest->monitor_mode == EXTERNAL_MODE && fw_mon->size) {
980e705c121SKalle Valo 		iwl_write_prph(trans, le32_to_cpu(dest->base_reg),
98169f0e505SShahar S Matityahu 			       fw_mon->physical >> dest->base_shift);
982286ca8ebSLuca Coelho 		if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
983e705c121SKalle Valo 			iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
98469f0e505SShahar S Matityahu 				       (fw_mon->physical + fw_mon->size -
98569f0e505SShahar S Matityahu 					256) >> dest->end_shift);
98662d7476dSEmmanuel Grumbach 		else
98762d7476dSEmmanuel Grumbach 			iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
98869f0e505SShahar S Matityahu 				       (fw_mon->physical + fw_mon->size) >>
98962d7476dSEmmanuel Grumbach 				       dest->end_shift);
990e705c121SKalle Valo 	}
991e705c121SKalle Valo }
992e705c121SKalle Valo 
993e705c121SKalle Valo static int iwl_pcie_load_given_ucode(struct iwl_trans *trans,
994e705c121SKalle Valo 				const struct fw_img *image)
995e705c121SKalle Valo {
996e705c121SKalle Valo 	int ret = 0;
997e705c121SKalle Valo 	int first_ucode_section;
998e705c121SKalle Valo 
999e705c121SKalle Valo 	IWL_DEBUG_FW(trans, "working with %s CPU\n",
1000e705c121SKalle Valo 		     image->is_dual_cpus ? "Dual" : "Single");
1001e705c121SKalle Valo 
1002e705c121SKalle Valo 	/* load to FW the binary non secured sections of CPU1 */
1003e705c121SKalle Valo 	ret = iwl_pcie_load_cpu_sections(trans, image, 1, &first_ucode_section);
1004e705c121SKalle Valo 	if (ret)
1005e705c121SKalle Valo 		return ret;
1006e705c121SKalle Valo 
1007e705c121SKalle Valo 	if (image->is_dual_cpus) {
1008e705c121SKalle Valo 		/* set CPU2 header address */
1009e705c121SKalle Valo 		iwl_write_prph(trans,
1010e705c121SKalle Valo 			       LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
1011e705c121SKalle Valo 			       LMPM_SECURE_CPU2_HDR_MEM_SPACE);
1012e705c121SKalle Valo 
1013e705c121SKalle Valo 		/* load to FW the binary sections of CPU2 */
1014e705c121SKalle Valo 		ret = iwl_pcie_load_cpu_sections(trans, image, 2,
1015e705c121SKalle Valo 						 &first_ucode_section);
1016e705c121SKalle Valo 		if (ret)
1017e705c121SKalle Valo 			return ret;
1018e705c121SKalle Valo 	}
1019e705c121SKalle Valo 
10209efab1adSEmmanuel Grumbach 	if (iwl_pcie_dbg_on(trans))
1021e705c121SKalle Valo 		iwl_pcie_apply_destination(trans);
1022e705c121SKalle Valo 
10232aabdbdcSEmmanuel Grumbach 	iwl_enable_interrupts(trans);
10242aabdbdcSEmmanuel Grumbach 
1025e705c121SKalle Valo 	/* release CPU reset */
1026e705c121SKalle Valo 	iwl_write32(trans, CSR_RESET, 0);
1027e705c121SKalle Valo 
1028e705c121SKalle Valo 	return 0;
1029e705c121SKalle Valo }
1030e705c121SKalle Valo 
1031e705c121SKalle Valo static int iwl_pcie_load_given_ucode_8000(struct iwl_trans *trans,
1032e705c121SKalle Valo 					  const struct fw_img *image)
1033e705c121SKalle Valo {
1034e705c121SKalle Valo 	int ret = 0;
1035e705c121SKalle Valo 	int first_ucode_section;
1036e705c121SKalle Valo 
1037e705c121SKalle Valo 	IWL_DEBUG_FW(trans, "working with %s CPU\n",
1038e705c121SKalle Valo 		     image->is_dual_cpus ? "Dual" : "Single");
1039e705c121SKalle Valo 
10407a14c23dSSara Sharon 	if (iwl_pcie_dbg_on(trans))
1041e705c121SKalle Valo 		iwl_pcie_apply_destination(trans);
1042e705c121SKalle Valo 
104382ea7966SSara Sharon 	IWL_DEBUG_POWER(trans, "Original WFPM value = 0x%08X\n",
104482ea7966SSara Sharon 			iwl_read_prph(trans, WFPM_GP2));
104582ea7966SSara Sharon 
104682ea7966SSara Sharon 	/*
104782ea7966SSara Sharon 	 * Set default value. On resume reading the values that were
104882ea7966SSara Sharon 	 * zeored can provide debug data on the resume flow.
104982ea7966SSara Sharon 	 * This is for debugging only and has no functional impact.
105082ea7966SSara Sharon 	 */
105182ea7966SSara Sharon 	iwl_write_prph(trans, WFPM_GP2, 0x01010101);
105282ea7966SSara Sharon 
1053e705c121SKalle Valo 	/* configure the ucode to be ready to get the secured image */
1054e705c121SKalle Valo 	/* release CPU reset */
1055e705c121SKalle Valo 	iwl_write_prph(trans, RELEASE_CPU_RESET, RELEASE_CPU_RESET_BIT);
1056e705c121SKalle Valo 
1057e705c121SKalle Valo 	/* load to FW the binary Secured sections of CPU1 */
1058e705c121SKalle Valo 	ret = iwl_pcie_load_cpu_sections_8000(trans, image, 1,
1059e705c121SKalle Valo 					      &first_ucode_section);
1060e705c121SKalle Valo 	if (ret)
1061e705c121SKalle Valo 		return ret;
1062e705c121SKalle Valo 
1063e705c121SKalle Valo 	/* load to FW the binary sections of CPU2 */
1064e705c121SKalle Valo 	return iwl_pcie_load_cpu_sections_8000(trans, image, 2,
1065e705c121SKalle Valo 					       &first_ucode_section);
1066e705c121SKalle Valo }
1067e705c121SKalle Valo 
10689ad8fd0bSJohannes Berg bool iwl_pcie_check_hw_rf_kill(struct iwl_trans *trans)
1069727c02dfSSara Sharon {
1070326477e4SJohannes Berg 	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1071727c02dfSSara Sharon 	bool hw_rfkill = iwl_is_rfkill_set(trans);
1072326477e4SJohannes Berg 	bool prev = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1073326477e4SJohannes Berg 	bool report;
1074727c02dfSSara Sharon 
1075326477e4SJohannes Berg 	if (hw_rfkill) {
1076326477e4SJohannes Berg 		set_bit(STATUS_RFKILL_HW, &trans->status);
1077326477e4SJohannes Berg 		set_bit(STATUS_RFKILL_OPMODE, &trans->status);
1078326477e4SJohannes Berg 	} else {
1079326477e4SJohannes Berg 		clear_bit(STATUS_RFKILL_HW, &trans->status);
1080326477e4SJohannes Berg 		if (trans_pcie->opmode_down)
1081326477e4SJohannes Berg 			clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
1082326477e4SJohannes Berg 	}
1083727c02dfSSara Sharon 
1084326477e4SJohannes Berg 	report = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1085326477e4SJohannes Berg 
1086326477e4SJohannes Berg 	if (prev != report)
1087326477e4SJohannes Berg 		iwl_trans_pcie_rf_kill(trans, report);
1088727c02dfSSara Sharon 
1089727c02dfSSara Sharon 	return hw_rfkill;
1090727c02dfSSara Sharon }
1091727c02dfSSara Sharon 
10927ca00409SHaim Dreyfuss struct iwl_causes_list {
10937ca00409SHaim Dreyfuss 	u32 cause_num;
10947ca00409SHaim Dreyfuss 	u32 mask_reg;
10957ca00409SHaim Dreyfuss 	u8 addr;
10967ca00409SHaim Dreyfuss };
10977ca00409SHaim Dreyfuss 
10987ca00409SHaim Dreyfuss static struct iwl_causes_list causes_list[] = {
10997ca00409SHaim Dreyfuss 	{MSIX_FH_INT_CAUSES_D2S_CH0_NUM,	CSR_MSIX_FH_INT_MASK_AD, 0},
11007ca00409SHaim Dreyfuss 	{MSIX_FH_INT_CAUSES_D2S_CH1_NUM,	CSR_MSIX_FH_INT_MASK_AD, 0x1},
11017ca00409SHaim Dreyfuss 	{MSIX_FH_INT_CAUSES_S2D,		CSR_MSIX_FH_INT_MASK_AD, 0x3},
11027ca00409SHaim Dreyfuss 	{MSIX_FH_INT_CAUSES_FH_ERR,		CSR_MSIX_FH_INT_MASK_AD, 0x5},
11037ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_ALIVE,		CSR_MSIX_HW_INT_MASK_AD, 0x10},
11047ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_WAKEUP,		CSR_MSIX_HW_INT_MASK_AD, 0x11},
1105ff911dcaSShaul Triebitz 	{MSIX_HW_INT_CAUSES_REG_IML,            CSR_MSIX_HW_INT_MASK_AD, 0x12},
11067ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_CT_KILL,	CSR_MSIX_HW_INT_MASK_AD, 0x16},
11077ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_RF_KILL,	CSR_MSIX_HW_INT_MASK_AD, 0x17},
11087ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_PERIODIC,	CSR_MSIX_HW_INT_MASK_AD, 0x18},
11097ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_SW_ERR,		CSR_MSIX_HW_INT_MASK_AD, 0x29},
11107ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_SCD,		CSR_MSIX_HW_INT_MASK_AD, 0x2A},
11117ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_FH_TX,		CSR_MSIX_HW_INT_MASK_AD, 0x2B},
11127ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_HW_ERR,		CSR_MSIX_HW_INT_MASK_AD, 0x2D},
11137ca00409SHaim Dreyfuss 	{MSIX_HW_INT_CAUSES_REG_HAP,		CSR_MSIX_HW_INT_MASK_AD, 0x2E},
11147ca00409SHaim Dreyfuss };
11157ca00409SHaim Dreyfuss 
11167ca00409SHaim Dreyfuss static void iwl_pcie_map_non_rx_causes(struct iwl_trans *trans)
11177ca00409SHaim Dreyfuss {
11187ca00409SHaim Dreyfuss 	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
11197ca00409SHaim Dreyfuss 	int val = trans_pcie->def_irq | MSIX_NON_AUTO_CLEAR_CAUSE;
11203681021fSJohannes Berg 	int i, arr_size = ARRAY_SIZE(causes_list);
11213681021fSJohannes Berg 	struct iwl_causes_list *causes = causes_list;
11227ca00409SHaim Dreyfuss 
11237ca00409SHaim Dreyfuss 	/*
11247ca00409SHaim Dreyfuss 	 * Access all non RX causes and map them to the default irq.
11257ca00409SHaim Dreyfuss 	 * In case we are missing at least one interrupt vector,
11267ca00409SHaim Dreyfuss 	 * the first interrupt vector will serve non-RX and FBQ causes.
11277ca00409SHaim Dreyfuss 	 */
11289b58419eSGolan Ben Ami 	for (i = 0; i < arr_size; i++) {
11299b58419eSGolan Ben Ami 		iwl_write8(trans, CSR_MSIX_IVAR(causes[i].addr), val);
11309b58419eSGolan Ben Ami 		iwl_clear_bit(trans, causes[i].mask_reg,
11319b58419eSGolan Ben Ami 			      causes[i].cause_num);
11327ca00409SHaim Dreyfuss 	}
11337ca00409SHaim Dreyfuss }
11347ca00409SHaim Dreyfuss 
11357ca00409SHaim Dreyfuss static void iwl_pcie_map_rx_causes(struct iwl_trans *trans)
11367ca00409SHaim Dreyfuss {
11377ca00409SHaim Dreyfuss 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
11387ca00409SHaim Dreyfuss 	u32 offset =
11397ca00409SHaim Dreyfuss 		trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 1 : 0;
11407ca00409SHaim Dreyfuss 	u32 val, idx;
11417ca00409SHaim Dreyfuss 
11427ca00409SHaim Dreyfuss 	/*
11437ca00409SHaim Dreyfuss 	 * The first RX queue - fallback queue, which is designated for
11447ca00409SHaim Dreyfuss 	 * management frame, command responses etc, is always mapped to the
11457ca00409SHaim Dreyfuss 	 * first interrupt vector. The other RX queues are mapped to
11467ca00409SHaim Dreyfuss 	 * the other (N - 2) interrupt vectors.
11477ca00409SHaim Dreyfuss 	 */
11487ca00409SHaim Dreyfuss 	val = BIT(MSIX_FH_INT_CAUSES_Q(0));
11497ca00409SHaim Dreyfuss 	for (idx = 1; idx < trans->num_rx_queues; idx++) {
11507ca00409SHaim Dreyfuss 		iwl_write8(trans, CSR_MSIX_RX_IVAR(idx),
11517ca00409SHaim Dreyfuss 			   MSIX_FH_INT_CAUSES_Q(idx - offset));
11527ca00409SHaim Dreyfuss 		val |= BIT(MSIX_FH_INT_CAUSES_Q(idx));
11537ca00409SHaim Dreyfuss 	}
11547ca00409SHaim Dreyfuss 	iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~val);
11557ca00409SHaim Dreyfuss 
11567ca00409SHaim Dreyfuss 	val = MSIX_FH_INT_CAUSES_Q(0);
11577ca00409SHaim Dreyfuss 	if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX)
11587ca00409SHaim Dreyfuss 		val |= MSIX_NON_AUTO_CLEAR_CAUSE;
11597ca00409SHaim Dreyfuss 	iwl_write8(trans, CSR_MSIX_RX_IVAR(0), val);
11607ca00409SHaim Dreyfuss 
11617ca00409SHaim Dreyfuss 	if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS)
11627ca00409SHaim Dreyfuss 		iwl_write8(trans, CSR_MSIX_RX_IVAR(1), val);
11637ca00409SHaim Dreyfuss }
11647ca00409SHaim Dreyfuss 
116577c09bc8SSara Sharon void iwl_pcie_conf_msix_hw(struct iwl_trans_pcie *trans_pcie)
11667ca00409SHaim Dreyfuss {
11677ca00409SHaim Dreyfuss 	struct iwl_trans *trans = trans_pcie->trans;
11687ca00409SHaim Dreyfuss 
11697ca00409SHaim Dreyfuss 	if (!trans_pcie->msix_enabled) {
1170286ca8ebSLuca Coelho 		if (trans->trans_cfg->mq_rx_supported &&
1171d7270d61SHaim Dreyfuss 		    test_bit(STATUS_DEVICE_ENABLED, &trans->status))
1172ea695b7cSShaul Triebitz 			iwl_write_umac_prph(trans, UREG_CHICK,
11737ca00409SHaim Dreyfuss 					    UREG_CHICK_MSI_ENABLE);
11747ca00409SHaim Dreyfuss 		return;
11757ca00409SHaim Dreyfuss 	}
1176d7270d61SHaim Dreyfuss 	/*
1177d7270d61SHaim Dreyfuss 	 * The IVAR table needs to be configured again after reset,
1178d7270d61SHaim Dreyfuss 	 * but if the device is disabled, we can't write to
1179d7270d61SHaim Dreyfuss 	 * prph.
1180d7270d61SHaim Dreyfuss 	 */
1181d7270d61SHaim Dreyfuss 	if (test_bit(STATUS_DEVICE_ENABLED, &trans->status))
1182ea695b7cSShaul Triebitz 		iwl_write_umac_prph(trans, UREG_CHICK, UREG_CHICK_MSIX_ENABLE);
11837ca00409SHaim Dreyfuss 
11847ca00409SHaim Dreyfuss 	/*
11857ca00409SHaim Dreyfuss 	 * Each cause from the causes list above and the RX causes is
11867ca00409SHaim Dreyfuss 	 * represented as a byte in the IVAR table. The first nibble
11877ca00409SHaim Dreyfuss 	 * represents the bound interrupt vector of the cause, the second
11887ca00409SHaim Dreyfuss 	 * represents no auto clear for this cause. This will be set if its
11897ca00409SHaim Dreyfuss 	 * interrupt vector is bound to serve other causes.
11907ca00409SHaim Dreyfuss 	 */
11917ca00409SHaim Dreyfuss 	iwl_pcie_map_rx_causes(trans);
11927ca00409SHaim Dreyfuss 
11937ca00409SHaim Dreyfuss 	iwl_pcie_map_non_rx_causes(trans);
119483730058SHaim Dreyfuss }
11957ca00409SHaim Dreyfuss 
119683730058SHaim Dreyfuss static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie)
119783730058SHaim Dreyfuss {
119883730058SHaim Dreyfuss 	struct iwl_trans *trans = trans_pcie->trans;
119983730058SHaim Dreyfuss 
120083730058SHaim Dreyfuss 	iwl_pcie_conf_msix_hw(trans_pcie);
120183730058SHaim Dreyfuss 
120283730058SHaim Dreyfuss 	if (!trans_pcie->msix_enabled)
120383730058SHaim Dreyfuss 		return;
120483730058SHaim Dreyfuss 
120583730058SHaim Dreyfuss 	trans_pcie->fh_init_mask = ~iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD);
12067ca00409SHaim Dreyfuss 	trans_pcie->fh_mask = trans_pcie->fh_init_mask;
120783730058SHaim Dreyfuss 	trans_pcie->hw_init_mask = ~iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD);
12087ca00409SHaim Dreyfuss 	trans_pcie->hw_mask = trans_pcie->hw_init_mask;
12097ca00409SHaim Dreyfuss }
12107ca00409SHaim Dreyfuss 
1211bab3cb92SEmmanuel Grumbach static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans)
1212e705c121SKalle Valo {
1213e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1214e705c121SKalle Valo 
1215e705c121SKalle Valo 	lockdep_assert_held(&trans_pcie->mutex);
1216e705c121SKalle Valo 
1217e705c121SKalle Valo 	if (trans_pcie->is_down)
1218e705c121SKalle Valo 		return;
1219e705c121SKalle Valo 
1220e705c121SKalle Valo 	trans_pcie->is_down = true;
1221e705c121SKalle Valo 
1222e705c121SKalle Valo 	/* tell the device to stop sending interrupts */
1223e705c121SKalle Valo 	iwl_disable_interrupts(trans);
1224e705c121SKalle Valo 
1225e705c121SKalle Valo 	/* device going down, Stop using ICT table */
1226e705c121SKalle Valo 	iwl_pcie_disable_ict(trans);
1227e705c121SKalle Valo 
1228e705c121SKalle Valo 	/*
1229e705c121SKalle Valo 	 * If a HW restart happens during firmware loading,
1230e705c121SKalle Valo 	 * then the firmware loading might call this function
1231e705c121SKalle Valo 	 * and later it might be called again due to the
1232e705c121SKalle Valo 	 * restart. So don't process again if the device is
1233e705c121SKalle Valo 	 * already dead.
1234e705c121SKalle Valo 	 */
1235e705c121SKalle Valo 	if (test_and_clear_bit(STATUS_DEVICE_ENABLED, &trans->status)) {
1236a6bd005fSEmmanuel Grumbach 		IWL_DEBUG_INFO(trans,
1237a6bd005fSEmmanuel Grumbach 			       "DEVICE_ENABLED bit was set and is now cleared\n");
1238e705c121SKalle Valo 		iwl_pcie_tx_stop(trans);
1239e705c121SKalle Valo 		iwl_pcie_rx_stop(trans);
1240e705c121SKalle Valo 
1241e705c121SKalle Valo 		/* Power-down device's busmaster DMA clocks */
1242e705c121SKalle Valo 		if (!trans->cfg->apmg_not_supported) {
1243e705c121SKalle Valo 			iwl_write_prph(trans, APMG_CLK_DIS_REG,
1244e705c121SKalle Valo 				       APMG_CLK_VAL_DMA_CLK_RQT);
1245e705c121SKalle Valo 			udelay(5);
1246e705c121SKalle Valo 		}
1247e705c121SKalle Valo 	}
1248e705c121SKalle Valo 
1249e705c121SKalle Valo 	/* Make sure (redundant) we've released our request to stay awake */
1250e705c121SKalle Valo 	iwl_clear_bit(trans, CSR_GP_CNTRL,
12516dece0e9SLuca Coelho 		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1252e705c121SKalle Valo 
1253e705c121SKalle Valo 	/* Stop the device, and put it in low power state */
1254e705c121SKalle Valo 	iwl_pcie_apm_stop(trans, false);
1255e705c121SKalle Valo 
1256870c2a11SGolan Ben Ami 	iwl_trans_pcie_sw_reset(trans);
1257e705c121SKalle Valo 
1258e705c121SKalle Valo 	/*
1259f4a1f04aSGolan Ben Ami 	 * Upon stop, the IVAR table gets erased, so msi-x won't
1260f4a1f04aSGolan Ben Ami 	 * work. This causes a bug in RF-KILL flows, since the interrupt
1261f4a1f04aSGolan Ben Ami 	 * that enables radio won't fire on the correct irq, and the
1262f4a1f04aSGolan Ben Ami 	 * driver won't be able to handle the interrupt.
1263f4a1f04aSGolan Ben Ami 	 * Configure the IVAR table again after reset.
1264f4a1f04aSGolan Ben Ami 	 */
1265f4a1f04aSGolan Ben Ami 	iwl_pcie_conf_msix_hw(trans_pcie);
1266f4a1f04aSGolan Ben Ami 
1267f4a1f04aSGolan Ben Ami 	/*
1268e705c121SKalle Valo 	 * Upon stop, the APM issues an interrupt if HW RF kill is set.
1269e705c121SKalle Valo 	 * This is a bug in certain verions of the hardware.
1270e705c121SKalle Valo 	 * Certain devices also keep sending HW RF kill interrupt all
1271e705c121SKalle Valo 	 * the time, unless the interrupt is ACKed even if the interrupt
1272e705c121SKalle Valo 	 * should be masked. Re-ACK all the interrupts here.
1273e705c121SKalle Valo 	 */
1274e705c121SKalle Valo 	iwl_disable_interrupts(trans);
1275e705c121SKalle Valo 
1276e705c121SKalle Valo 	/* clear all status bits */
1277e705c121SKalle Valo 	clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1278e705c121SKalle Valo 	clear_bit(STATUS_INT_ENABLED, &trans->status);
1279e705c121SKalle Valo 	clear_bit(STATUS_TPOWER_PMI, &trans->status);
1280e705c121SKalle Valo 
1281e705c121SKalle Valo 	/*
1282e705c121SKalle Valo 	 * Even if we stop the HW, we still want the RF kill
1283e705c121SKalle Valo 	 * interrupt
1284e705c121SKalle Valo 	 */
1285e705c121SKalle Valo 	iwl_enable_rfkill_int(trans);
1286e705c121SKalle Valo 
1287a6bd005fSEmmanuel Grumbach 	/* re-take ownership to prevent other users from stealing the device */
1288e705c121SKalle Valo 	iwl_pcie_prepare_card_hw(trans);
1289e705c121SKalle Valo }
1290e705c121SKalle Valo 
1291eda50cdeSSara Sharon void iwl_pcie_synchronize_irqs(struct iwl_trans *trans)
12922e5d4a8fSHaim Dreyfuss {
12932e5d4a8fSHaim Dreyfuss 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
12942e5d4a8fSHaim Dreyfuss 
12952e5d4a8fSHaim Dreyfuss 	if (trans_pcie->msix_enabled) {
12962e5d4a8fSHaim Dreyfuss 		int i;
12972e5d4a8fSHaim Dreyfuss 
1298496d83caSHaim Dreyfuss 		for (i = 0; i < trans_pcie->alloc_vecs; i++)
12992e5d4a8fSHaim Dreyfuss 			synchronize_irq(trans_pcie->msix_entries[i].vector);
13002e5d4a8fSHaim Dreyfuss 	} else {
13012e5d4a8fSHaim Dreyfuss 		synchronize_irq(trans_pcie->pci_dev->irq);
13022e5d4a8fSHaim Dreyfuss 	}
13032e5d4a8fSHaim Dreyfuss }
13042e5d4a8fSHaim Dreyfuss 
1305a6bd005fSEmmanuel Grumbach static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
1306a6bd005fSEmmanuel Grumbach 				   const struct fw_img *fw, bool run_in_rfkill)
1307a6bd005fSEmmanuel Grumbach {
1308a6bd005fSEmmanuel Grumbach 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1309a6bd005fSEmmanuel Grumbach 	bool hw_rfkill;
1310a6bd005fSEmmanuel Grumbach 	int ret;
1311a6bd005fSEmmanuel Grumbach 
1312a6bd005fSEmmanuel Grumbach 	/* This may fail if AMT took ownership of the device */
1313a6bd005fSEmmanuel Grumbach 	if (iwl_pcie_prepare_card_hw(trans)) {
1314a6bd005fSEmmanuel Grumbach 		IWL_WARN(trans, "Exit HW not ready\n");
1315a6bd005fSEmmanuel Grumbach 		ret = -EIO;
1316a6bd005fSEmmanuel Grumbach 		goto out;
1317a6bd005fSEmmanuel Grumbach 	}
1318a6bd005fSEmmanuel Grumbach 
1319a6bd005fSEmmanuel Grumbach 	iwl_enable_rfkill_int(trans);
1320a6bd005fSEmmanuel Grumbach 
1321a6bd005fSEmmanuel Grumbach 	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1322a6bd005fSEmmanuel Grumbach 
1323a6bd005fSEmmanuel Grumbach 	/*
1324a6bd005fSEmmanuel Grumbach 	 * We enabled the RF-Kill interrupt and the handler may very
1325a6bd005fSEmmanuel Grumbach 	 * well be running. Disable the interrupts to make sure no other
1326a6bd005fSEmmanuel Grumbach 	 * interrupt can be fired.
1327a6bd005fSEmmanuel Grumbach 	 */
1328a6bd005fSEmmanuel Grumbach 	iwl_disable_interrupts(trans);
1329a6bd005fSEmmanuel Grumbach 
1330a6bd005fSEmmanuel Grumbach 	/* Make sure it finished running */
13312e5d4a8fSHaim Dreyfuss 	iwl_pcie_synchronize_irqs(trans);
1332a6bd005fSEmmanuel Grumbach 
1333a6bd005fSEmmanuel Grumbach 	mutex_lock(&trans_pcie->mutex);
1334a6bd005fSEmmanuel Grumbach 
1335a6bd005fSEmmanuel Grumbach 	/* If platform's RF_KILL switch is NOT set to KILL */
13369ad8fd0bSJohannes Berg 	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
1337a6bd005fSEmmanuel Grumbach 	if (hw_rfkill && !run_in_rfkill) {
1338a6bd005fSEmmanuel Grumbach 		ret = -ERFKILL;
1339a6bd005fSEmmanuel Grumbach 		goto out;
1340a6bd005fSEmmanuel Grumbach 	}
1341a6bd005fSEmmanuel Grumbach 
1342a6bd005fSEmmanuel Grumbach 	/* Someone called stop_device, don't try to start_fw */
1343a6bd005fSEmmanuel Grumbach 	if (trans_pcie->is_down) {
1344a6bd005fSEmmanuel Grumbach 		IWL_WARN(trans,
1345a6bd005fSEmmanuel Grumbach 			 "Can't start_fw since the HW hasn't been started\n");
134620aa99bbSAnton Protopopov 		ret = -EIO;
1347a6bd005fSEmmanuel Grumbach 		goto out;
1348a6bd005fSEmmanuel Grumbach 	}
1349a6bd005fSEmmanuel Grumbach 
1350a6bd005fSEmmanuel Grumbach 	/* make sure rfkill handshake bits are cleared */
1351a6bd005fSEmmanuel Grumbach 	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1352a6bd005fSEmmanuel Grumbach 	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR,
1353a6bd005fSEmmanuel Grumbach 		    CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
1354a6bd005fSEmmanuel Grumbach 
1355a6bd005fSEmmanuel Grumbach 	/* clear (again), then enable host interrupts */
1356a6bd005fSEmmanuel Grumbach 	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1357a6bd005fSEmmanuel Grumbach 
1358a6bd005fSEmmanuel Grumbach 	ret = iwl_pcie_nic_init(trans);
1359a6bd005fSEmmanuel Grumbach 	if (ret) {
1360a6bd005fSEmmanuel Grumbach 		IWL_ERR(trans, "Unable to init nic\n");
1361a6bd005fSEmmanuel Grumbach 		goto out;
1362a6bd005fSEmmanuel Grumbach 	}
1363a6bd005fSEmmanuel Grumbach 
1364a6bd005fSEmmanuel Grumbach 	/*
1365a6bd005fSEmmanuel Grumbach 	 * Now, we load the firmware and don't want to be interrupted, even
1366a6bd005fSEmmanuel Grumbach 	 * by the RF-Kill interrupt (hence mask all the interrupt besides the
1367a6bd005fSEmmanuel Grumbach 	 * FH_TX interrupt which is needed to load the firmware). If the
1368a6bd005fSEmmanuel Grumbach 	 * RF-Kill switch is toggled, we will find out after having loaded
1369a6bd005fSEmmanuel Grumbach 	 * the firmware and return the proper value to the caller.
1370a6bd005fSEmmanuel Grumbach 	 */
1371a6bd005fSEmmanuel Grumbach 	iwl_enable_fw_load_int(trans);
1372a6bd005fSEmmanuel Grumbach 
1373a6bd005fSEmmanuel Grumbach 	/* really make sure rfkill handshake bits are cleared */
1374a6bd005fSEmmanuel Grumbach 	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1375a6bd005fSEmmanuel Grumbach 	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1376a6bd005fSEmmanuel Grumbach 
1377a6bd005fSEmmanuel Grumbach 	/* Load the given image to the HW */
1378286ca8ebSLuca Coelho 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
1379a6bd005fSEmmanuel Grumbach 		ret = iwl_pcie_load_given_ucode_8000(trans, fw);
1380a6bd005fSEmmanuel Grumbach 	else
1381a6bd005fSEmmanuel Grumbach 		ret = iwl_pcie_load_given_ucode(trans, fw);
1382a6bd005fSEmmanuel Grumbach 
1383a6bd005fSEmmanuel Grumbach 	/* re-check RF-Kill state since we may have missed the interrupt */
13849ad8fd0bSJohannes Berg 	hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
1385a6bd005fSEmmanuel Grumbach 	if (hw_rfkill && !run_in_rfkill)
1386a6bd005fSEmmanuel Grumbach 		ret = -ERFKILL;
1387a6bd005fSEmmanuel Grumbach 
1388a6bd005fSEmmanuel Grumbach out:
1389a6bd005fSEmmanuel Grumbach 	mutex_unlock(&trans_pcie->mutex);
1390a6bd005fSEmmanuel Grumbach 	return ret;
1391a6bd005fSEmmanuel Grumbach }
1392a6bd005fSEmmanuel Grumbach 
1393a6bd005fSEmmanuel Grumbach static void iwl_trans_pcie_fw_alive(struct iwl_trans *trans, u32 scd_addr)
1394a6bd005fSEmmanuel Grumbach {
1395a6bd005fSEmmanuel Grumbach 	iwl_pcie_reset_ict(trans);
1396a6bd005fSEmmanuel Grumbach 	iwl_pcie_tx_start(trans, scd_addr);
1397a6bd005fSEmmanuel Grumbach }
1398a6bd005fSEmmanuel Grumbach 
1399326477e4SJohannes Berg void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans,
1400326477e4SJohannes Berg 				       bool was_in_rfkill)
1401326477e4SJohannes Berg {
1402326477e4SJohannes Berg 	bool hw_rfkill;
1403326477e4SJohannes Berg 
1404326477e4SJohannes Berg 	/*
1405326477e4SJohannes Berg 	 * Check again since the RF kill state may have changed while
1406326477e4SJohannes Berg 	 * all the interrupts were disabled, in this case we couldn't
1407326477e4SJohannes Berg 	 * receive the RF kill interrupt and update the state in the
1408326477e4SJohannes Berg 	 * op_mode.
1409326477e4SJohannes Berg 	 * Don't call the op_mode if the rkfill state hasn't changed.
1410326477e4SJohannes Berg 	 * This allows the op_mode to call stop_device from the rfkill
1411326477e4SJohannes Berg 	 * notification without endless recursion. Under very rare
1412326477e4SJohannes Berg 	 * circumstances, we might have a small recursion if the rfkill
1413326477e4SJohannes Berg 	 * state changed exactly now while we were called from stop_device.
1414326477e4SJohannes Berg 	 * This is very unlikely but can happen and is supported.
1415326477e4SJohannes Berg 	 */
1416326477e4SJohannes Berg 	hw_rfkill = iwl_is_rfkill_set(trans);
1417326477e4SJohannes Berg 	if (hw_rfkill) {
1418326477e4SJohannes Berg 		set_bit(STATUS_RFKILL_HW, &trans->status);
1419326477e4SJohannes Berg 		set_bit(STATUS_RFKILL_OPMODE, &trans->status);
1420326477e4SJohannes Berg 	} else {
1421326477e4SJohannes Berg 		clear_bit(STATUS_RFKILL_HW, &trans->status);
1422326477e4SJohannes Berg 		clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
1423326477e4SJohannes Berg 	}
1424326477e4SJohannes Berg 	if (hw_rfkill != was_in_rfkill)
1425326477e4SJohannes Berg 		iwl_trans_pcie_rf_kill(trans, hw_rfkill);
1426326477e4SJohannes Berg }
1427326477e4SJohannes Berg 
1428bab3cb92SEmmanuel Grumbach static void iwl_trans_pcie_stop_device(struct iwl_trans *trans)
1429e705c121SKalle Valo {
1430e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1431326477e4SJohannes Berg 	bool was_in_rfkill;
1432e705c121SKalle Valo 
1433e705c121SKalle Valo 	mutex_lock(&trans_pcie->mutex);
1434326477e4SJohannes Berg 	trans_pcie->opmode_down = true;
1435326477e4SJohannes Berg 	was_in_rfkill = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1436bab3cb92SEmmanuel Grumbach 	_iwl_trans_pcie_stop_device(trans);
1437326477e4SJohannes Berg 	iwl_trans_pcie_handle_stop_rfkill(trans, was_in_rfkill);
1438e705c121SKalle Valo 	mutex_unlock(&trans_pcie->mutex);
1439e705c121SKalle Valo }
1440e705c121SKalle Valo 
1441e705c121SKalle Valo void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state)
1442e705c121SKalle Valo {
1443e705c121SKalle Valo 	struct iwl_trans_pcie __maybe_unused *trans_pcie =
1444e705c121SKalle Valo 		IWL_TRANS_GET_PCIE_TRANS(trans);
1445e705c121SKalle Valo 
1446e705c121SKalle Valo 	lockdep_assert_held(&trans_pcie->mutex);
1447e705c121SKalle Valo 
1448326477e4SJohannes Berg 	IWL_WARN(trans, "reporting RF_KILL (radio %s)\n",
1449326477e4SJohannes Berg 		 state ? "disabled" : "enabled");
145077c09bc8SSara Sharon 	if (iwl_op_mode_hw_rf_kill(trans->op_mode, state)) {
1451286ca8ebSLuca Coelho 		if (trans->trans_cfg->gen2)
1452bab3cb92SEmmanuel Grumbach 			_iwl_trans_pcie_gen2_stop_device(trans);
145377c09bc8SSara Sharon 		else
1454bab3cb92SEmmanuel Grumbach 			_iwl_trans_pcie_stop_device(trans);
1455e705c121SKalle Valo 	}
145677c09bc8SSara Sharon }
1457e705c121SKalle Valo 
1458e5f3f215SHaim Dreyfuss void iwl_pcie_d3_complete_suspend(struct iwl_trans *trans,
1459e5f3f215SHaim Dreyfuss 				  bool test, bool reset)
1460e705c121SKalle Valo {
1461e705c121SKalle Valo 	iwl_disable_interrupts(trans);
1462e705c121SKalle Valo 
1463e705c121SKalle Valo 	/*
1464e705c121SKalle Valo 	 * in testing mode, the host stays awake and the
1465e705c121SKalle Valo 	 * hardware won't be reset (not even partially)
1466e705c121SKalle Valo 	 */
1467e705c121SKalle Valo 	if (test)
1468e705c121SKalle Valo 		return;
1469e705c121SKalle Valo 
1470e705c121SKalle Valo 	iwl_pcie_disable_ict(trans);
1471e705c121SKalle Valo 
14722e5d4a8fSHaim Dreyfuss 	iwl_pcie_synchronize_irqs(trans);
1473e705c121SKalle Valo 
1474e705c121SKalle Valo 	iwl_clear_bit(trans, CSR_GP_CNTRL,
14756dece0e9SLuca Coelho 		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
14766dece0e9SLuca Coelho 	iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
1477e705c121SKalle Valo 
147823ae6128SMatti Gottlieb 	if (reset) {
1479e705c121SKalle Valo 		/*
1480e705c121SKalle Valo 		 * reset TX queues -- some of their registers reset during S3
1481e705c121SKalle Valo 		 * so if we don't reset everything here the D3 image would try
1482e705c121SKalle Valo 		 * to execute some invalid memory upon resume
1483e705c121SKalle Valo 		 */
1484e705c121SKalle Valo 		iwl_trans_pcie_tx_reset(trans);
1485e705c121SKalle Valo 	}
1486e705c121SKalle Valo 
1487e705c121SKalle Valo 	iwl_pcie_set_pwr(trans, true);
1488e705c121SKalle Valo }
1489e705c121SKalle Valo 
1490e5f3f215SHaim Dreyfuss static int iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test,
1491e5f3f215SHaim Dreyfuss 				     bool reset)
1492e5f3f215SHaim Dreyfuss {
1493e5f3f215SHaim Dreyfuss 	int ret;
1494e5f3f215SHaim Dreyfuss 	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1495e5f3f215SHaim Dreyfuss 
1496771db3a1SHaim Dreyfuss 	if (!reset)
1497e5f3f215SHaim Dreyfuss 		/* Enable persistence mode to avoid reset */
1498e5f3f215SHaim Dreyfuss 		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
1499e5f3f215SHaim Dreyfuss 			    CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
1500e5f3f215SHaim Dreyfuss 
1501e5f3f215SHaim Dreyfuss 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
1502e5f3f215SHaim Dreyfuss 		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
1503e5f3f215SHaim Dreyfuss 				    UREG_DOORBELL_TO_ISR6_SUSPEND);
1504e5f3f215SHaim Dreyfuss 
1505e5f3f215SHaim Dreyfuss 		ret = wait_event_timeout(trans_pcie->sx_waitq,
1506e5f3f215SHaim Dreyfuss 					 trans_pcie->sx_complete, 2 * HZ);
1507e5f3f215SHaim Dreyfuss 		/*
1508e5f3f215SHaim Dreyfuss 		 * Invalidate it toward resume.
1509e5f3f215SHaim Dreyfuss 		 */
1510e5f3f215SHaim Dreyfuss 		trans_pcie->sx_complete = false;
1511e5f3f215SHaim Dreyfuss 
1512e5f3f215SHaim Dreyfuss 		if (!ret) {
1513e5f3f215SHaim Dreyfuss 			IWL_ERR(trans, "Timeout entering D3\n");
1514e5f3f215SHaim Dreyfuss 			return -ETIMEDOUT;
1515e5f3f215SHaim Dreyfuss 		}
1516e5f3f215SHaim Dreyfuss 	}
1517e5f3f215SHaim Dreyfuss 	iwl_pcie_d3_complete_suspend(trans, test, reset);
1518e5f3f215SHaim Dreyfuss 
1519e5f3f215SHaim Dreyfuss 	return 0;
1520e5f3f215SHaim Dreyfuss }
1521e5f3f215SHaim Dreyfuss 
1522e705c121SKalle Valo static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
1523e705c121SKalle Valo 				    enum iwl_d3_status *status,
152423ae6128SMatti Gottlieb 				    bool test,  bool reset)
1525e705c121SKalle Valo {
1526d7270d61SHaim Dreyfuss 	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1527e705c121SKalle Valo 	u32 val;
1528e705c121SKalle Valo 	int ret;
1529e705c121SKalle Valo 
1530e705c121SKalle Valo 	if (test) {
1531e705c121SKalle Valo 		iwl_enable_interrupts(trans);
1532e705c121SKalle Valo 		*status = IWL_D3_STATUS_ALIVE;
1533e5f3f215SHaim Dreyfuss 		goto out;
1534e705c121SKalle Valo 	}
1535e705c121SKalle Valo 
1536a8cbb46fSGolan Ben Ami 	iwl_set_bit(trans, CSR_GP_CNTRL,
15376dece0e9SLuca Coelho 		    CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1538e705c121SKalle Valo 
15397d34a7d7SLuca Coelho 	ret = iwl_finish_nic_init(trans, trans->trans_cfg);
1540c96b5eecSJohannes Berg 	if (ret)
1541e705c121SKalle Valo 		return ret;
1542e705c121SKalle Valo 
1543f98ad635SEmmanuel Grumbach 	/*
1544f98ad635SEmmanuel Grumbach 	 * Reconfigure IVAR table in case of MSIX or reset ict table in
1545f98ad635SEmmanuel Grumbach 	 * MSI mode since HW reset erased it.
1546f98ad635SEmmanuel Grumbach 	 * Also enables interrupts - none will happen as
1547f98ad635SEmmanuel Grumbach 	 * the device doesn't know we're waking it up, only when
1548f98ad635SEmmanuel Grumbach 	 * the opmode actually tells it after this call.
1549f98ad635SEmmanuel Grumbach 	 */
1550f98ad635SEmmanuel Grumbach 	iwl_pcie_conf_msix_hw(trans_pcie);
1551f98ad635SEmmanuel Grumbach 	if (!trans_pcie->msix_enabled)
1552f98ad635SEmmanuel Grumbach 		iwl_pcie_reset_ict(trans);
1553f98ad635SEmmanuel Grumbach 	iwl_enable_interrupts(trans);
1554f98ad635SEmmanuel Grumbach 
1555e705c121SKalle Valo 	iwl_pcie_set_pwr(trans, false);
1556e705c121SKalle Valo 
155723ae6128SMatti Gottlieb 	if (!reset) {
1558e705c121SKalle Valo 		iwl_clear_bit(trans, CSR_GP_CNTRL,
15596dece0e9SLuca Coelho 			      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1560e705c121SKalle Valo 	} else {
1561e705c121SKalle Valo 		iwl_trans_pcie_tx_reset(trans);
1562e705c121SKalle Valo 
1563e705c121SKalle Valo 		ret = iwl_pcie_rx_init(trans);
1564e705c121SKalle Valo 		if (ret) {
1565e705c121SKalle Valo 			IWL_ERR(trans,
1566e705c121SKalle Valo 				"Failed to resume the device (RX reset)\n");
1567e705c121SKalle Valo 			return ret;
1568e705c121SKalle Valo 		}
1569e705c121SKalle Valo 	}
1570e705c121SKalle Valo 
157182ea7966SSara Sharon 	IWL_DEBUG_POWER(trans, "WFPM value upon resume = 0x%08X\n",
1572ea695b7cSShaul Triebitz 			iwl_read_umac_prph(trans, WFPM_GP2));
157382ea7966SSara Sharon 
1574e705c121SKalle Valo 	val = iwl_read32(trans, CSR_RESET);
1575e705c121SKalle Valo 	if (val & CSR_RESET_REG_FLAG_NEVO_RESET)
1576e705c121SKalle Valo 		*status = IWL_D3_STATUS_RESET;
1577e705c121SKalle Valo 	else
1578e705c121SKalle Valo 		*status = IWL_D3_STATUS_ALIVE;
1579e705c121SKalle Valo 
1580e5f3f215SHaim Dreyfuss out:
1581e5f3f215SHaim Dreyfuss 	if (*status == IWL_D3_STATUS_ALIVE &&
1582e5f3f215SHaim Dreyfuss 	    trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
1583e5f3f215SHaim Dreyfuss 		trans_pcie->sx_complete = false;
1584e5f3f215SHaim Dreyfuss 		iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
1585e5f3f215SHaim Dreyfuss 				    UREG_DOORBELL_TO_ISR6_RESUME);
1586e5f3f215SHaim Dreyfuss 
1587e5f3f215SHaim Dreyfuss 		ret = wait_event_timeout(trans_pcie->sx_waitq,
1588e5f3f215SHaim Dreyfuss 					 trans_pcie->sx_complete, 2 * HZ);
1589e5f3f215SHaim Dreyfuss 		/*
1590e5f3f215SHaim Dreyfuss 		 * Invalidate it toward next suspend.
1591e5f3f215SHaim Dreyfuss 		 */
1592e5f3f215SHaim Dreyfuss 		trans_pcie->sx_complete = false;
1593e5f3f215SHaim Dreyfuss 
1594e5f3f215SHaim Dreyfuss 		if (!ret) {
1595e5f3f215SHaim Dreyfuss 			IWL_ERR(trans, "Timeout exiting D3\n");
1596e5f3f215SHaim Dreyfuss 			return -ETIMEDOUT;
1597e5f3f215SHaim Dreyfuss 		}
1598e5f3f215SHaim Dreyfuss 	}
1599e705c121SKalle Valo 	return 0;
1600e705c121SKalle Valo }
1601e705c121SKalle Valo 
16020c18714aSLuca Coelho static void
16030c18714aSLuca Coelho iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
16040c18714aSLuca Coelho 			    struct iwl_trans *trans,
16050c18714aSLuca Coelho 			    const struct iwl_cfg_trans_params *cfg_trans)
16062e5d4a8fSHaim Dreyfuss {
16072e5d4a8fSHaim Dreyfuss 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1608ab1068d6SHao Wei Tee 	int max_irqs, num_irqs, i, ret;
16092e5d4a8fSHaim Dreyfuss 	u16 pci_cmd;
16100cd38f4dSMordechay Goodstein 	u32 max_rx_queues = IWL_MAX_RX_HW_QUEUES;
16112e5d4a8fSHaim Dreyfuss 
16120c18714aSLuca Coelho 	if (!cfg_trans->mq_rx_supported)
161306f4b081SSara Sharon 		goto enable_msi;
161406f4b081SSara Sharon 
16150cd38f4dSMordechay Goodstein 	if (cfg_trans->device_family <= IWL_DEVICE_FAMILY_9000)
16160cd38f4dSMordechay Goodstein 		max_rx_queues = IWL_9000_MAX_RX_HW_QUEUES;
16170cd38f4dSMordechay Goodstein 
16180cd38f4dSMordechay Goodstein 	max_irqs = min_t(u32, num_online_cpus() + 2, max_rx_queues);
161906f4b081SSara Sharon 	for (i = 0; i < max_irqs; i++)
16202e5d4a8fSHaim Dreyfuss 		trans_pcie->msix_entries[i].entry = i;
16212e5d4a8fSHaim Dreyfuss 
162206f4b081SSara Sharon 	num_irqs = pci_enable_msix_range(pdev, trans_pcie->msix_entries,
16232e5d4a8fSHaim Dreyfuss 					 MSIX_MIN_INTERRUPT_VECTORS,
162406f4b081SSara Sharon 					 max_irqs);
162506f4b081SSara Sharon 	if (num_irqs < 0) {
1626496d83caSHaim Dreyfuss 		IWL_DEBUG_INFO(trans,
162706f4b081SSara Sharon 			       "Failed to enable msi-x mode (ret %d). Moving to msi mode.\n",
162806f4b081SSara Sharon 			       num_irqs);
162906f4b081SSara Sharon 		goto enable_msi;
1630496d83caSHaim Dreyfuss 	}
163106f4b081SSara Sharon 	trans_pcie->def_irq = (num_irqs == max_irqs) ? num_irqs - 1 : 0;
1632496d83caSHaim Dreyfuss 
16332e5d4a8fSHaim Dreyfuss 	IWL_DEBUG_INFO(trans,
163406f4b081SSara Sharon 		       "MSI-X enabled. %d interrupt vectors were allocated\n",
163506f4b081SSara Sharon 		       num_irqs);
163606f4b081SSara Sharon 
1637496d83caSHaim Dreyfuss 	/*
163806f4b081SSara Sharon 	 * In case the OS provides fewer interrupts than requested, different
163906f4b081SSara Sharon 	 * causes will share the same interrupt vector as follows:
1640496d83caSHaim Dreyfuss 	 * One interrupt less: non rx causes shared with FBQ.
1641496d83caSHaim Dreyfuss 	 * Two interrupts less: non rx causes shared with FBQ and RSS.
1642496d83caSHaim Dreyfuss 	 * More than two interrupts: we will use fewer RSS queues.
1643496d83caSHaim Dreyfuss 	 */
1644ab1068d6SHao Wei Tee 	if (num_irqs <= max_irqs - 2) {
164506f4b081SSara Sharon 		trans_pcie->trans->num_rx_queues = num_irqs + 1;
1646496d83caSHaim Dreyfuss 		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX |
1647496d83caSHaim Dreyfuss 			IWL_SHARED_IRQ_FIRST_RSS;
1648ab1068d6SHao Wei Tee 	} else if (num_irqs == max_irqs - 1) {
164906f4b081SSara Sharon 		trans_pcie->trans->num_rx_queues = num_irqs;
1650496d83caSHaim Dreyfuss 		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX;
1651496d83caSHaim Dreyfuss 	} else {
165206f4b081SSara Sharon 		trans_pcie->trans->num_rx_queues = num_irqs - 1;
1653496d83caSHaim Dreyfuss 	}
1654ab1068d6SHao Wei Tee 	WARN_ON(trans_pcie->trans->num_rx_queues > IWL_MAX_RX_HW_QUEUES);
16552e5d4a8fSHaim Dreyfuss 
165606f4b081SSara Sharon 	trans_pcie->alloc_vecs = num_irqs;
1657496d83caSHaim Dreyfuss 	trans_pcie->msix_enabled = true;
16582e5d4a8fSHaim Dreyfuss 	return;
16592e5d4a8fSHaim Dreyfuss 
166006f4b081SSara Sharon enable_msi:
166106f4b081SSara Sharon 	ret = pci_enable_msi(pdev);
166206f4b081SSara Sharon 	if (ret) {
166306f4b081SSara Sharon 		dev_err(&pdev->dev, "pci_enable_msi failed - %d\n", ret);
16642e5d4a8fSHaim Dreyfuss 		/* enable rfkill interrupt: hw bug w/a */
16652e5d4a8fSHaim Dreyfuss 		pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
16662e5d4a8fSHaim Dreyfuss 		if (pci_cmd & PCI_COMMAND_INTX_DISABLE) {
16672e5d4a8fSHaim Dreyfuss 			pci_cmd &= ~PCI_COMMAND_INTX_DISABLE;
16682e5d4a8fSHaim Dreyfuss 			pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
16692e5d4a8fSHaim Dreyfuss 		}
16702e5d4a8fSHaim Dreyfuss 	}
16712e5d4a8fSHaim Dreyfuss }
16722e5d4a8fSHaim Dreyfuss 
16737c8d91ebSHaim Dreyfuss static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans)
16747c8d91ebSHaim Dreyfuss {
16757c8d91ebSHaim Dreyfuss 	int iter_rx_q, i, ret, cpu, offset;
16767c8d91ebSHaim Dreyfuss 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
16777c8d91ebSHaim Dreyfuss 
16787c8d91ebSHaim Dreyfuss 	i = trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 0 : 1;
16797c8d91ebSHaim Dreyfuss 	iter_rx_q = trans_pcie->trans->num_rx_queues - 1 + i;
16807c8d91ebSHaim Dreyfuss 	offset = 1 + i;
16817c8d91ebSHaim Dreyfuss 	for (; i < iter_rx_q ; i++) {
16827c8d91ebSHaim Dreyfuss 		/*
16837c8d91ebSHaim Dreyfuss 		 * Get the cpu prior to the place to search
16847c8d91ebSHaim Dreyfuss 		 * (i.e. return will be > i - 1).
16857c8d91ebSHaim Dreyfuss 		 */
16867c8d91ebSHaim Dreyfuss 		cpu = cpumask_next(i - offset, cpu_online_mask);
16877c8d91ebSHaim Dreyfuss 		cpumask_set_cpu(cpu, &trans_pcie->affinity_mask[i]);
16887c8d91ebSHaim Dreyfuss 		ret = irq_set_affinity_hint(trans_pcie->msix_entries[i].vector,
16897c8d91ebSHaim Dreyfuss 					    &trans_pcie->affinity_mask[i]);
16907c8d91ebSHaim Dreyfuss 		if (ret)
16917c8d91ebSHaim Dreyfuss 			IWL_ERR(trans_pcie->trans,
16927c8d91ebSHaim Dreyfuss 				"Failed to set affinity mask for IRQ %d\n",
16937c8d91ebSHaim Dreyfuss 				i);
16947c8d91ebSHaim Dreyfuss 	}
16957c8d91ebSHaim Dreyfuss }
16967c8d91ebSHaim Dreyfuss 
16972e5d4a8fSHaim Dreyfuss static int iwl_pcie_init_msix_handler(struct pci_dev *pdev,
16982e5d4a8fSHaim Dreyfuss 				      struct iwl_trans_pcie *trans_pcie)
16992e5d4a8fSHaim Dreyfuss {
1700496d83caSHaim Dreyfuss 	int i;
17012e5d4a8fSHaim Dreyfuss 
1702496d83caSHaim Dreyfuss 	for (i = 0; i < trans_pcie->alloc_vecs; i++) {
17032e5d4a8fSHaim Dreyfuss 		int ret;
17045a41a86cSSharon Dvir 		struct msix_entry *msix_entry;
170564fa3affSSharon Dvir 		const char *qname = queue_name(&pdev->dev, trans_pcie, i);
170664fa3affSSharon Dvir 
170764fa3affSSharon Dvir 		if (!qname)
170864fa3affSSharon Dvir 			return -ENOMEM;
17092e5d4a8fSHaim Dreyfuss 
17105a41a86cSSharon Dvir 		msix_entry = &trans_pcie->msix_entries[i];
17115a41a86cSSharon Dvir 		ret = devm_request_threaded_irq(&pdev->dev,
17125a41a86cSSharon Dvir 						msix_entry->vector,
17132e5d4a8fSHaim Dreyfuss 						iwl_pcie_msix_isr,
1714496d83caSHaim Dreyfuss 						(i == trans_pcie->def_irq) ?
17152e5d4a8fSHaim Dreyfuss 						iwl_pcie_irq_msix_handler :
17162e5d4a8fSHaim Dreyfuss 						iwl_pcie_irq_rx_msix_handler,
17172e5d4a8fSHaim Dreyfuss 						IRQF_SHARED,
171864fa3affSSharon Dvir 						qname,
17195a41a86cSSharon Dvir 						msix_entry);
17202e5d4a8fSHaim Dreyfuss 		if (ret) {
17212e5d4a8fSHaim Dreyfuss 			IWL_ERR(trans_pcie->trans,
17222e5d4a8fSHaim Dreyfuss 				"Error allocating IRQ %d\n", i);
17235a41a86cSSharon Dvir 
17242e5d4a8fSHaim Dreyfuss 			return ret;
17252e5d4a8fSHaim Dreyfuss 		}
17262e5d4a8fSHaim Dreyfuss 	}
17277c8d91ebSHaim Dreyfuss 	iwl_pcie_irq_set_affinity(trans_pcie->trans);
17282e5d4a8fSHaim Dreyfuss 
17292e5d4a8fSHaim Dreyfuss 	return 0;
17302e5d4a8fSHaim Dreyfuss }
17312e5d4a8fSHaim Dreyfuss 
173244f61b5cSShahar S Matityahu static int iwl_trans_pcie_clear_persistence_bit(struct iwl_trans *trans)
173344f61b5cSShahar S Matityahu {
173444f61b5cSShahar S Matityahu 	u32 hpm, wprot;
173544f61b5cSShahar S Matityahu 
1736286ca8ebSLuca Coelho 	switch (trans->trans_cfg->device_family) {
173744f61b5cSShahar S Matityahu 	case IWL_DEVICE_FAMILY_9000:
173844f61b5cSShahar S Matityahu 		wprot = PREG_PRPH_WPROT_9000;
173944f61b5cSShahar S Matityahu 		break;
174044f61b5cSShahar S Matityahu 	case IWL_DEVICE_FAMILY_22000:
174144f61b5cSShahar S Matityahu 		wprot = PREG_PRPH_WPROT_22000;
174244f61b5cSShahar S Matityahu 		break;
174344f61b5cSShahar S Matityahu 	default:
174444f61b5cSShahar S Matityahu 		return 0;
174544f61b5cSShahar S Matityahu 	}
174644f61b5cSShahar S Matityahu 
174744f61b5cSShahar S Matityahu 	hpm = iwl_read_umac_prph_no_grab(trans, HPM_DEBUG);
174844f61b5cSShahar S Matityahu 	if (hpm != 0xa5a5a5a0 && (hpm & PERSISTENCE_BIT)) {
174944f61b5cSShahar S Matityahu 		u32 wprot_val = iwl_read_umac_prph_no_grab(trans, wprot);
175044f61b5cSShahar S Matityahu 
175144f61b5cSShahar S Matityahu 		if (wprot_val & PREG_WFPM_ACCESS) {
175244f61b5cSShahar S Matityahu 			IWL_ERR(trans,
175344f61b5cSShahar S Matityahu 				"Error, can not clear persistence bit\n");
175444f61b5cSShahar S Matityahu 			return -EPERM;
175544f61b5cSShahar S Matityahu 		}
175644f61b5cSShahar S Matityahu 		iwl_write_umac_prph_no_grab(trans, HPM_DEBUG,
175744f61b5cSShahar S Matityahu 					    hpm & ~PERSISTENCE_BIT);
175844f61b5cSShahar S Matityahu 	}
175944f61b5cSShahar S Matityahu 
176044f61b5cSShahar S Matityahu 	return 0;
176144f61b5cSShahar S Matityahu }
176244f61b5cSShahar S Matityahu 
17630df36b90SLuca Coelho static int iwl_pcie_gen2_force_power_gating(struct iwl_trans *trans)
17640df36b90SLuca Coelho {
17650df36b90SLuca Coelho 	int ret;
17660df36b90SLuca Coelho 
17670df36b90SLuca Coelho 	ret = iwl_finish_nic_init(trans, trans->trans_cfg);
17680df36b90SLuca Coelho 	if (ret < 0)
17690df36b90SLuca Coelho 		return ret;
17700df36b90SLuca Coelho 
17710df36b90SLuca Coelho 	iwl_set_bits_prph(trans, HPM_HIPM_GEN_CFG,
17720df36b90SLuca Coelho 			  HPM_HIPM_GEN_CFG_CR_FORCE_ACTIVE);
17730df36b90SLuca Coelho 	udelay(20);
17740df36b90SLuca Coelho 	iwl_set_bits_prph(trans, HPM_HIPM_GEN_CFG,
17750df36b90SLuca Coelho 			  HPM_HIPM_GEN_CFG_CR_PG_EN |
17760df36b90SLuca Coelho 			  HPM_HIPM_GEN_CFG_CR_SLP_EN);
17770df36b90SLuca Coelho 	udelay(20);
17780df36b90SLuca Coelho 	iwl_clear_bits_prph(trans, HPM_HIPM_GEN_CFG,
17790df36b90SLuca Coelho 			    HPM_HIPM_GEN_CFG_CR_FORCE_ACTIVE);
17800df36b90SLuca Coelho 
17810df36b90SLuca Coelho 	iwl_trans_pcie_sw_reset(trans);
17820df36b90SLuca Coelho 
17830df36b90SLuca Coelho 	return 0;
17840df36b90SLuca Coelho }
17850df36b90SLuca Coelho 
1786bab3cb92SEmmanuel Grumbach static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans)
1787e705c121SKalle Valo {
1788e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1789e705c121SKalle Valo 	int err;
1790e705c121SKalle Valo 
1791e705c121SKalle Valo 	lockdep_assert_held(&trans_pcie->mutex);
1792e705c121SKalle Valo 
1793e705c121SKalle Valo 	err = iwl_pcie_prepare_card_hw(trans);
1794e705c121SKalle Valo 	if (err) {
1795e705c121SKalle Valo 		IWL_ERR(trans, "Error while preparing HW: %d\n", err);
1796e705c121SKalle Valo 		return err;
1797e705c121SKalle Valo 	}
1798e705c121SKalle Valo 
179944f61b5cSShahar S Matityahu 	err = iwl_trans_pcie_clear_persistence_bit(trans);
180044f61b5cSShahar S Matityahu 	if (err)
180144f61b5cSShahar S Matityahu 		return err;
18028954e1ebSShahar S Matityahu 
1803870c2a11SGolan Ben Ami 	iwl_trans_pcie_sw_reset(trans);
1804e705c121SKalle Valo 
18050df36b90SLuca Coelho 	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000 &&
18067897dfa2SLuca Coelho 	    trans->trans_cfg->integrated) {
18070df36b90SLuca Coelho 		err = iwl_pcie_gen2_force_power_gating(trans);
18080df36b90SLuca Coelho 		if (err)
18090df36b90SLuca Coelho 			return err;
18100df36b90SLuca Coelho 	}
18110df36b90SLuca Coelho 
181252b6e168SEmmanuel Grumbach 	err = iwl_pcie_apm_init(trans);
181352b6e168SEmmanuel Grumbach 	if (err)
181452b6e168SEmmanuel Grumbach 		return err;
1815e705c121SKalle Valo 
18162e5d4a8fSHaim Dreyfuss 	iwl_pcie_init_msix(trans_pcie);
181783730058SHaim Dreyfuss 
1818e705c121SKalle Valo 	/* From now on, the op_mode will be kept updated about RF kill state */
1819e705c121SKalle Valo 	iwl_enable_rfkill_int(trans);
1820e705c121SKalle Valo 
1821326477e4SJohannes Berg 	trans_pcie->opmode_down = false;
1822326477e4SJohannes Berg 
1823e705c121SKalle Valo 	/* Set is_down to false here so that...*/
1824e705c121SKalle Valo 	trans_pcie->is_down = false;
1825e705c121SKalle Valo 
1826e705c121SKalle Valo 	/* ...rfkill can call stop_device and set it false if needed */
18279ad8fd0bSJohannes Berg 	iwl_pcie_check_hw_rf_kill(trans);
1828e705c121SKalle Valo 
1829e705c121SKalle Valo 	return 0;
1830e705c121SKalle Valo }
1831e705c121SKalle Valo 
1832bab3cb92SEmmanuel Grumbach static int iwl_trans_pcie_start_hw(struct iwl_trans *trans)
1833e705c121SKalle Valo {
1834e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1835e705c121SKalle Valo 	int ret;
1836e705c121SKalle Valo 
1837e705c121SKalle Valo 	mutex_lock(&trans_pcie->mutex);
1838bab3cb92SEmmanuel Grumbach 	ret = _iwl_trans_pcie_start_hw(trans);
1839e705c121SKalle Valo 	mutex_unlock(&trans_pcie->mutex);
1840e705c121SKalle Valo 
1841e705c121SKalle Valo 	return ret;
1842e705c121SKalle Valo }
1843e705c121SKalle Valo 
1844e705c121SKalle Valo static void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans)
1845e705c121SKalle Valo {
1846e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1847e705c121SKalle Valo 
1848e705c121SKalle Valo 	mutex_lock(&trans_pcie->mutex);
1849e705c121SKalle Valo 
1850e705c121SKalle Valo 	/* disable interrupts - don't enable HW RF kill interrupt */
1851e705c121SKalle Valo 	iwl_disable_interrupts(trans);
1852e705c121SKalle Valo 
1853e705c121SKalle Valo 	iwl_pcie_apm_stop(trans, true);
1854e705c121SKalle Valo 
1855e705c121SKalle Valo 	iwl_disable_interrupts(trans);
1856e705c121SKalle Valo 
1857e705c121SKalle Valo 	iwl_pcie_disable_ict(trans);
1858e705c121SKalle Valo 
1859e705c121SKalle Valo 	mutex_unlock(&trans_pcie->mutex);
1860e705c121SKalle Valo 
18612e5d4a8fSHaim Dreyfuss 	iwl_pcie_synchronize_irqs(trans);
1862e705c121SKalle Valo }
1863e705c121SKalle Valo 
1864e705c121SKalle Valo static void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val)
1865e705c121SKalle Valo {
1866e705c121SKalle Valo 	writeb(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1867e705c121SKalle Valo }
1868e705c121SKalle Valo 
1869e705c121SKalle Valo static void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val)
1870e705c121SKalle Valo {
1871e705c121SKalle Valo 	writel(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1872e705c121SKalle Valo }
1873e705c121SKalle Valo 
1874e705c121SKalle Valo static u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs)
1875e705c121SKalle Valo {
1876e705c121SKalle Valo 	return readl(IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1877e705c121SKalle Valo }
1878e705c121SKalle Valo 
187984fb372cSSara Sharon static u32 iwl_trans_pcie_prph_msk(struct iwl_trans *trans)
188084fb372cSSara Sharon {
18813681021fSJohannes Berg 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
188284fb372cSSara Sharon 		return 0x00FFFFFF;
188384fb372cSSara Sharon 	else
188484fb372cSSara Sharon 		return 0x000FFFFF;
188584fb372cSSara Sharon }
188684fb372cSSara Sharon 
1887e705c121SKalle Valo static u32 iwl_trans_pcie_read_prph(struct iwl_trans *trans, u32 reg)
1888e705c121SKalle Valo {
188984fb372cSSara Sharon 	u32 mask = iwl_trans_pcie_prph_msk(trans);
189084fb372cSSara Sharon 
1891e705c121SKalle Valo 	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_RADDR,
189284fb372cSSara Sharon 			       ((reg & mask) | (3 << 24)));
1893e705c121SKalle Valo 	return iwl_trans_pcie_read32(trans, HBUS_TARG_PRPH_RDAT);
1894e705c121SKalle Valo }
1895e705c121SKalle Valo 
1896e705c121SKalle Valo static void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr,
1897e705c121SKalle Valo 				      u32 val)
1898e705c121SKalle Valo {
189984fb372cSSara Sharon 	u32 mask = iwl_trans_pcie_prph_msk(trans);
190084fb372cSSara Sharon 
1901e705c121SKalle Valo 	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WADDR,
190284fb372cSSara Sharon 			       ((addr & mask) | (3 << 24)));
1903e705c121SKalle Valo 	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WDAT, val);
1904e705c121SKalle Valo }
1905e705c121SKalle Valo 
1906e705c121SKalle Valo static void iwl_trans_pcie_configure(struct iwl_trans *trans,
1907e705c121SKalle Valo 				     const struct iwl_trans_config *trans_cfg)
1908e705c121SKalle Valo {
1909e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1910e705c121SKalle Valo 
19114f4822b7SMordechay Goodstein 	trans->txqs.cmd.q_id = trans_cfg->cmd_queue;
19124f4822b7SMordechay Goodstein 	trans->txqs.cmd.fifo = trans_cfg->cmd_fifo;
19134f4822b7SMordechay Goodstein 	trans->txqs.cmd.wdg_timeout = trans_cfg->cmd_q_wdg_timeout;
191422852fadSMordechay Goodstein 	trans->txqs.page_offs = trans_cfg->cb_data_offs;
191522852fadSMordechay Goodstein 	trans->txqs.dev_cmd_offs = trans_cfg->cb_data_offs + sizeof(void *);
191622852fadSMordechay Goodstein 
1917e705c121SKalle Valo 	if (WARN_ON(trans_cfg->n_no_reclaim_cmds > MAX_NO_RECLAIM_CMDS))
1918e705c121SKalle Valo 		trans_pcie->n_no_reclaim_cmds = 0;
1919e705c121SKalle Valo 	else
1920e705c121SKalle Valo 		trans_pcie->n_no_reclaim_cmds = trans_cfg->n_no_reclaim_cmds;
1921e705c121SKalle Valo 	if (trans_pcie->n_no_reclaim_cmds)
1922e705c121SKalle Valo 		memcpy(trans_pcie->no_reclaim_cmds, trans_cfg->no_reclaim_cmds,
1923e705c121SKalle Valo 		       trans_pcie->n_no_reclaim_cmds * sizeof(u8));
1924e705c121SKalle Valo 
19256c4fbcbcSEmmanuel Grumbach 	trans_pcie->rx_buf_size = trans_cfg->rx_buf_size;
19266c4fbcbcSEmmanuel Grumbach 	trans_pcie->rx_page_order =
19276c4fbcbcSEmmanuel Grumbach 		iwl_trans_get_rb_size_order(trans_pcie->rx_buf_size);
192880084e35SJohannes Berg 	trans_pcie->rx_buf_bytes =
192980084e35SJohannes Berg 		iwl_trans_get_rb_size(trans_pcie->rx_buf_size);
1930cfdc20efSJohannes Berg 	trans_pcie->supported_dma_mask = DMA_BIT_MASK(12);
1931cfdc20efSJohannes Berg 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
1932cfdc20efSJohannes Berg 		trans_pcie->supported_dma_mask = DMA_BIT_MASK(11);
1933e705c121SKalle Valo 
19348e3b79f8SMordechay Goodstein 	trans->txqs.bc_table_dword = trans_cfg->bc_table_dword;
1935e705c121SKalle Valo 	trans_pcie->scd_set_active = trans_cfg->scd_set_active;
193641837ca9SEmmanuel Grumbach 	trans_pcie->sw_csum_tx = trans_cfg->sw_csum_tx;
1937e705c121SKalle Valo 
193839bdb17eSSharon Dvir 	trans->command_groups = trans_cfg->command_groups;
193939bdb17eSSharon Dvir 	trans->command_groups_size = trans_cfg->command_groups_size;
194039bdb17eSSharon Dvir 
1941e705c121SKalle Valo 	/* Initialize NAPI here - it should be before registering to mac80211
1942e705c121SKalle Valo 	 * in the opmode but after the HW struct is allocated.
1943e705c121SKalle Valo 	 * As this function may be called again in some corner cases don't
1944e705c121SKalle Valo 	 * do anything if NAPI was already initialized.
1945e705c121SKalle Valo 	 */
1946bce97731SSara Sharon 	if (trans_pcie->napi_dev.reg_state != NETREG_DUMMY)
1947e705c121SKalle Valo 		init_dummy_netdev(&trans_pcie->napi_dev);
1948e705c121SKalle Valo }
1949e705c121SKalle Valo 
1950e705c121SKalle Valo void iwl_trans_pcie_free(struct iwl_trans *trans)
1951e705c121SKalle Valo {
1952e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
19536eb5e529SEmmanuel Grumbach 	int i;
1954e705c121SKalle Valo 
19552e5d4a8fSHaim Dreyfuss 	iwl_pcie_synchronize_irqs(trans);
1956e705c121SKalle Valo 
1957286ca8ebSLuca Coelho 	if (trans->trans_cfg->gen2)
19580cd1ad2dSMordechay Goodstein 		iwl_txq_gen2_tx_free(trans);
195913a3a390SSara Sharon 	else
1960e705c121SKalle Valo 		iwl_pcie_tx_free(trans);
1961e705c121SKalle Valo 	iwl_pcie_rx_free(trans);
1962e705c121SKalle Valo 
196310a54d81SLuca Coelho 	if (trans_pcie->rba.alloc_wq) {
196410a54d81SLuca Coelho 		destroy_workqueue(trans_pcie->rba.alloc_wq);
196510a54d81SLuca Coelho 		trans_pcie->rba.alloc_wq = NULL;
196610a54d81SLuca Coelho 	}
196710a54d81SLuca Coelho 
19682e5d4a8fSHaim Dreyfuss 	if (trans_pcie->msix_enabled) {
19697c8d91ebSHaim Dreyfuss 		for (i = 0; i < trans_pcie->alloc_vecs; i++) {
19707c8d91ebSHaim Dreyfuss 			irq_set_affinity_hint(
19717c8d91ebSHaim Dreyfuss 				trans_pcie->msix_entries[i].vector,
19727c8d91ebSHaim Dreyfuss 				NULL);
19737c8d91ebSHaim Dreyfuss 		}
19742e5d4a8fSHaim Dreyfuss 
19752e5d4a8fSHaim Dreyfuss 		trans_pcie->msix_enabled = false;
19762e5d4a8fSHaim Dreyfuss 	} else {
1977e705c121SKalle Valo 		iwl_pcie_free_ict(trans);
19782e5d4a8fSHaim Dreyfuss 	}
1979e705c121SKalle Valo 
1980e705c121SKalle Valo 	iwl_pcie_free_fw_monitor(trans);
1981e705c121SKalle Valo 
1982a2a57a35SEmmanuel Grumbach 	mutex_destroy(&trans_pcie->mutex);
1983e705c121SKalle Valo 	iwl_trans_free(trans);
1984e705c121SKalle Valo }
1985e705c121SKalle Valo 
1986e705c121SKalle Valo static void iwl_trans_pcie_set_pmi(struct iwl_trans *trans, bool state)
1987e705c121SKalle Valo {
1988e705c121SKalle Valo 	if (state)
1989e705c121SKalle Valo 		set_bit(STATUS_TPOWER_PMI, &trans->status);
1990e705c121SKalle Valo 	else
1991e705c121SKalle Valo 		clear_bit(STATUS_TPOWER_PMI, &trans->status);
1992e705c121SKalle Valo }
1993e705c121SKalle Valo 
199449564a80SLuca Coelho struct iwl_trans_pcie_removal {
199549564a80SLuca Coelho 	struct pci_dev *pdev;
199649564a80SLuca Coelho 	struct work_struct work;
199749564a80SLuca Coelho };
199849564a80SLuca Coelho 
199949564a80SLuca Coelho static void iwl_trans_pcie_removal_wk(struct work_struct *wk)
200049564a80SLuca Coelho {
200149564a80SLuca Coelho 	struct iwl_trans_pcie_removal *removal =
200249564a80SLuca Coelho 		container_of(wk, struct iwl_trans_pcie_removal, work);
200349564a80SLuca Coelho 	struct pci_dev *pdev = removal->pdev;
2004aba1e632SColin Ian King 	static char *prop[] = {"EVENT=INACCESSIBLE", NULL};
200549564a80SLuca Coelho 
200649564a80SLuca Coelho 	dev_err(&pdev->dev, "Device gone - attempting removal\n");
200749564a80SLuca Coelho 	kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, prop);
200849564a80SLuca Coelho 	pci_lock_rescan_remove();
200949564a80SLuca Coelho 	pci_dev_put(pdev);
201049564a80SLuca Coelho 	pci_stop_and_remove_bus_device(pdev);
201149564a80SLuca Coelho 	pci_unlock_rescan_remove();
201249564a80SLuca Coelho 
201349564a80SLuca Coelho 	kfree(removal);
201449564a80SLuca Coelho 	module_put(THIS_MODULE);
201549564a80SLuca Coelho }
201649564a80SLuca Coelho 
201723ba9340SEmmanuel Grumbach static bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans,
2018e705c121SKalle Valo 					   unsigned long *flags)
2019e705c121SKalle Valo {
2020e705c121SKalle Valo 	int ret;
2021e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2022e705c121SKalle Valo 
2023e705c121SKalle Valo 	spin_lock_irqsave(&trans_pcie->reg_lock, *flags);
2024e705c121SKalle Valo 
2025e705c121SKalle Valo 	if (trans_pcie->cmd_hold_nic_awake)
2026e705c121SKalle Valo 		goto out;
2027e705c121SKalle Valo 
2028e705c121SKalle Valo 	/* this bit wakes up the NIC */
2029e705c121SKalle Valo 	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
20306dece0e9SLuca Coelho 				 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
2031286ca8ebSLuca Coelho 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
2032e705c121SKalle Valo 		udelay(2);
2033e705c121SKalle Valo 
2034e705c121SKalle Valo 	/*
2035e705c121SKalle Valo 	 * These bits say the device is running, and should keep running for
2036e705c121SKalle Valo 	 * at least a short while (at least as long as MAC_ACCESS_REQ stays 1),
2037e705c121SKalle Valo 	 * but they do not indicate that embedded SRAM is restored yet;
2038fb70d49fSLuca Coelho 	 * HW with volatile SRAM must save/restore contents to/from
2039fb70d49fSLuca Coelho 	 * host DRAM when sleeping/waking for power-saving.
2040e705c121SKalle Valo 	 * Each direction takes approximately 1/4 millisecond; with this
2041e705c121SKalle Valo 	 * overhead, it's a good idea to grab and hold MAC_ACCESS_REQUEST if a
2042e705c121SKalle Valo 	 * series of register accesses are expected (e.g. reading Event Log),
2043e705c121SKalle Valo 	 * to keep device from sleeping.
2044e705c121SKalle Valo 	 *
2045e705c121SKalle Valo 	 * CSR_UCODE_DRV_GP1 register bit MAC_SLEEP == 0 indicates that
2046e705c121SKalle Valo 	 * SRAM is okay/restored.  We don't check that here because this call
2047fb70d49fSLuca Coelho 	 * is just for hardware register access; but GP1 MAC_SLEEP
2048fb70d49fSLuca Coelho 	 * check is a good idea before accessing the SRAM of HW with
2049fb70d49fSLuca Coelho 	 * volatile SRAM (e.g. reading Event Log).
2050e705c121SKalle Valo 	 *
2051e705c121SKalle Valo 	 * 5000 series and later (including 1000 series) have non-volatile SRAM,
2052e705c121SKalle Valo 	 * and do not save/restore SRAM when power cycling.
2053e705c121SKalle Valo 	 */
2054e705c121SKalle Valo 	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
20556dece0e9SLuca Coelho 			   CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN,
20566dece0e9SLuca Coelho 			   (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
2057e705c121SKalle Valo 			    CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP), 15000);
2058e705c121SKalle Valo 	if (unlikely(ret < 0)) {
205949564a80SLuca Coelho 		u32 cntrl = iwl_read32(trans, CSR_GP_CNTRL);
206049564a80SLuca Coelho 
2061e705c121SKalle Valo 		WARN_ONCE(1,
2062e705c121SKalle Valo 			  "Timeout waiting for hardware access (CSR_GP_CNTRL 0x%08x)\n",
206349564a80SLuca Coelho 			  cntrl);
206449564a80SLuca Coelho 
206549564a80SLuca Coelho 		iwl_trans_pcie_dump_regs(trans);
206649564a80SLuca Coelho 
206749564a80SLuca Coelho 		if (iwlwifi_mod_params.remove_when_gone && cntrl == ~0U) {
206849564a80SLuca Coelho 			struct iwl_trans_pcie_removal *removal;
206949564a80SLuca Coelho 
2070f60c9e59SEmmanuel Grumbach 			if (test_bit(STATUS_TRANS_DEAD, &trans->status))
207149564a80SLuca Coelho 				goto err;
207249564a80SLuca Coelho 
207349564a80SLuca Coelho 			IWL_ERR(trans, "Device gone - scheduling removal!\n");
207449564a80SLuca Coelho 
207549564a80SLuca Coelho 			/*
207649564a80SLuca Coelho 			 * get a module reference to avoid doing this
207749564a80SLuca Coelho 			 * while unloading anyway and to avoid
207849564a80SLuca Coelho 			 * scheduling a work with code that's being
207949564a80SLuca Coelho 			 * removed.
208049564a80SLuca Coelho 			 */
208149564a80SLuca Coelho 			if (!try_module_get(THIS_MODULE)) {
208249564a80SLuca Coelho 				IWL_ERR(trans,
208349564a80SLuca Coelho 					"Module is being unloaded - abort\n");
208449564a80SLuca Coelho 				goto err;
208549564a80SLuca Coelho 			}
208649564a80SLuca Coelho 
208749564a80SLuca Coelho 			removal = kzalloc(sizeof(*removal), GFP_ATOMIC);
208849564a80SLuca Coelho 			if (!removal) {
208949564a80SLuca Coelho 				module_put(THIS_MODULE);
209049564a80SLuca Coelho 				goto err;
209149564a80SLuca Coelho 			}
209249564a80SLuca Coelho 			/*
209349564a80SLuca Coelho 			 * we don't need to clear this flag, because
209449564a80SLuca Coelho 			 * the trans will be freed and reallocated.
209549564a80SLuca Coelho 			*/
2096f60c9e59SEmmanuel Grumbach 			set_bit(STATUS_TRANS_DEAD, &trans->status);
209749564a80SLuca Coelho 
209849564a80SLuca Coelho 			removal->pdev = to_pci_dev(trans->dev);
209949564a80SLuca Coelho 			INIT_WORK(&removal->work, iwl_trans_pcie_removal_wk);
210049564a80SLuca Coelho 			pci_dev_get(removal->pdev);
210149564a80SLuca Coelho 			schedule_work(&removal->work);
210249564a80SLuca Coelho 		} else {
210349564a80SLuca Coelho 			iwl_write32(trans, CSR_RESET,
210449564a80SLuca Coelho 				    CSR_RESET_REG_FLAG_FORCE_NMI);
210549564a80SLuca Coelho 		}
210649564a80SLuca Coelho 
210749564a80SLuca Coelho err:
2108e705c121SKalle Valo 		spin_unlock_irqrestore(&trans_pcie->reg_lock, *flags);
2109e705c121SKalle Valo 		return false;
2110e705c121SKalle Valo 	}
2111e705c121SKalle Valo 
2112e705c121SKalle Valo out:
2113e705c121SKalle Valo 	/*
2114e705c121SKalle Valo 	 * Fool sparse by faking we release the lock - sparse will
2115e705c121SKalle Valo 	 * track nic_access anyway.
2116e705c121SKalle Valo 	 */
2117e705c121SKalle Valo 	__release(&trans_pcie->reg_lock);
2118e705c121SKalle Valo 	return true;
2119e705c121SKalle Valo }
2120e705c121SKalle Valo 
2121e705c121SKalle Valo static void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans,
2122e705c121SKalle Valo 					      unsigned long *flags)
2123e705c121SKalle Valo {
2124e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2125e705c121SKalle Valo 
2126e705c121SKalle Valo 	lockdep_assert_held(&trans_pcie->reg_lock);
2127e705c121SKalle Valo 
2128e705c121SKalle Valo 	/*
2129e705c121SKalle Valo 	 * Fool sparse by faking we acquiring the lock - sparse will
2130e705c121SKalle Valo 	 * track nic_access anyway.
2131e705c121SKalle Valo 	 */
2132e705c121SKalle Valo 	__acquire(&trans_pcie->reg_lock);
2133e705c121SKalle Valo 
2134e705c121SKalle Valo 	if (trans_pcie->cmd_hold_nic_awake)
2135e705c121SKalle Valo 		goto out;
2136e705c121SKalle Valo 
2137e705c121SKalle Valo 	__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
21386dece0e9SLuca Coelho 				   CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
2139e705c121SKalle Valo 	/*
2140e705c121SKalle Valo 	 * Above we read the CSR_GP_CNTRL register, which will flush
2141e705c121SKalle Valo 	 * any previous writes, but we need the write that clears the
2142e705c121SKalle Valo 	 * MAC_ACCESS_REQ bit to be performed before any other writes
2143e705c121SKalle Valo 	 * scheduled on different CPUs (after we drop reg_lock).
2144e705c121SKalle Valo 	 */
2145e705c121SKalle Valo out:
2146e705c121SKalle Valo 	spin_unlock_irqrestore(&trans_pcie->reg_lock, *flags);
2147e705c121SKalle Valo }
2148e705c121SKalle Valo 
2149e705c121SKalle Valo static int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
2150e705c121SKalle Valo 				   void *buf, int dwords)
2151e705c121SKalle Valo {
2152e705c121SKalle Valo 	unsigned long flags;
2153e705c121SKalle Valo 	int offs, ret = 0;
2154e705c121SKalle Valo 	u32 *vals = buf;
2155e705c121SKalle Valo 
215623ba9340SEmmanuel Grumbach 	if (iwl_trans_grab_nic_access(trans, &flags)) {
2157e705c121SKalle Valo 		iwl_write32(trans, HBUS_TARG_MEM_RADDR, addr);
2158e705c121SKalle Valo 		for (offs = 0; offs < dwords; offs++)
2159e705c121SKalle Valo 			vals[offs] = iwl_read32(trans, HBUS_TARG_MEM_RDAT);
2160e705c121SKalle Valo 		iwl_trans_release_nic_access(trans, &flags);
2161e705c121SKalle Valo 	} else {
2162e705c121SKalle Valo 		ret = -EBUSY;
2163e705c121SKalle Valo 	}
2164e705c121SKalle Valo 	return ret;
2165e705c121SKalle Valo }
2166e705c121SKalle Valo 
2167e705c121SKalle Valo static int iwl_trans_pcie_write_mem(struct iwl_trans *trans, u32 addr,
2168e705c121SKalle Valo 				    const void *buf, int dwords)
2169e705c121SKalle Valo {
2170e705c121SKalle Valo 	unsigned long flags;
2171e705c121SKalle Valo 	int offs, ret = 0;
2172e705c121SKalle Valo 	const u32 *vals = buf;
2173e705c121SKalle Valo 
217423ba9340SEmmanuel Grumbach 	if (iwl_trans_grab_nic_access(trans, &flags)) {
2175e705c121SKalle Valo 		iwl_write32(trans, HBUS_TARG_MEM_WADDR, addr);
2176e705c121SKalle Valo 		for (offs = 0; offs < dwords; offs++)
2177e705c121SKalle Valo 			iwl_write32(trans, HBUS_TARG_MEM_WDAT,
2178e705c121SKalle Valo 				    vals ? vals[offs] : 0);
2179e705c121SKalle Valo 		iwl_trans_release_nic_access(trans, &flags);
2180e705c121SKalle Valo 	} else {
2181e705c121SKalle Valo 		ret = -EBUSY;
2182e705c121SKalle Valo 	}
2183e705c121SKalle Valo 	return ret;
2184e705c121SKalle Valo }
2185e705c121SKalle Valo 
21867f1fe1d4SLuca Coelho static int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs,
21877f1fe1d4SLuca Coelho 					u32 *val)
21887f1fe1d4SLuca Coelho {
21897f1fe1d4SLuca Coelho 	return pci_read_config_dword(IWL_TRANS_GET_PCIE_TRANS(trans)->pci_dev,
21907f1fe1d4SLuca Coelho 				     ofs, val);
21917f1fe1d4SLuca Coelho }
21927f1fe1d4SLuca Coelho 
2193e705c121SKalle Valo static void iwl_trans_pcie_freeze_txq_timer(struct iwl_trans *trans,
2194e705c121SKalle Valo 					    unsigned long txqs,
2195e705c121SKalle Valo 					    bool freeze)
2196e705c121SKalle Valo {
2197e705c121SKalle Valo 	int queue;
2198e705c121SKalle Valo 
2199e705c121SKalle Valo 	for_each_set_bit(queue, &txqs, BITS_PER_LONG) {
22004f4822b7SMordechay Goodstein 		struct iwl_txq *txq = trans->txqs.txq[queue];
2201e705c121SKalle Valo 		unsigned long now;
2202e705c121SKalle Valo 
2203e705c121SKalle Valo 		spin_lock_bh(&txq->lock);
2204e705c121SKalle Valo 
2205e705c121SKalle Valo 		now = jiffies;
2206e705c121SKalle Valo 
2207e705c121SKalle Valo 		if (txq->frozen == freeze)
2208e705c121SKalle Valo 			goto next_queue;
2209e705c121SKalle Valo 
2210e705c121SKalle Valo 		IWL_DEBUG_TX_QUEUES(trans, "%s TXQ %d\n",
2211e705c121SKalle Valo 				    freeze ? "Freezing" : "Waking", queue);
2212e705c121SKalle Valo 
2213e705c121SKalle Valo 		txq->frozen = freeze;
2214e705c121SKalle Valo 
2215bb98ecd4SSara Sharon 		if (txq->read_ptr == txq->write_ptr)
2216e705c121SKalle Valo 			goto next_queue;
2217e705c121SKalle Valo 
2218e705c121SKalle Valo 		if (freeze) {
2219e705c121SKalle Valo 			if (unlikely(time_after(now,
2220e705c121SKalle Valo 						txq->stuck_timer.expires))) {
2221e705c121SKalle Valo 				/*
2222e705c121SKalle Valo 				 * The timer should have fired, maybe it is
2223e705c121SKalle Valo 				 * spinning right now on the lock.
2224e705c121SKalle Valo 				 */
2225e705c121SKalle Valo 				goto next_queue;
2226e705c121SKalle Valo 			}
2227e705c121SKalle Valo 			/* remember how long until the timer fires */
2228e705c121SKalle Valo 			txq->frozen_expiry_remainder =
2229e705c121SKalle Valo 				txq->stuck_timer.expires - now;
2230e705c121SKalle Valo 			del_timer(&txq->stuck_timer);
2231e705c121SKalle Valo 			goto next_queue;
2232e705c121SKalle Valo 		}
2233e705c121SKalle Valo 
2234e705c121SKalle Valo 		/*
2235e705c121SKalle Valo 		 * Wake a non-empty queue -> arm timer with the
2236e705c121SKalle Valo 		 * remainder before it froze
2237e705c121SKalle Valo 		 */
2238e705c121SKalle Valo 		mod_timer(&txq->stuck_timer,
2239e705c121SKalle Valo 			  now + txq->frozen_expiry_remainder);
2240e705c121SKalle Valo 
2241e705c121SKalle Valo next_queue:
2242e705c121SKalle Valo 		spin_unlock_bh(&txq->lock);
2243e705c121SKalle Valo 	}
2244e705c121SKalle Valo }
2245e705c121SKalle Valo 
22460cd58eaaSEmmanuel Grumbach static void iwl_trans_pcie_block_txq_ptrs(struct iwl_trans *trans, bool block)
22470cd58eaaSEmmanuel Grumbach {
22480cd58eaaSEmmanuel Grumbach 	int i;
22490cd58eaaSEmmanuel Grumbach 
2250286ca8ebSLuca Coelho 	for (i = 0; i < trans->trans_cfg->base_params->num_of_queues; i++) {
22514f4822b7SMordechay Goodstein 		struct iwl_txq *txq = trans->txqs.txq[i];
22520cd58eaaSEmmanuel Grumbach 
22534f4822b7SMordechay Goodstein 		if (i == trans->txqs.cmd.q_id)
22540cd58eaaSEmmanuel Grumbach 			continue;
22550cd58eaaSEmmanuel Grumbach 
22560cd58eaaSEmmanuel Grumbach 		spin_lock_bh(&txq->lock);
22570cd58eaaSEmmanuel Grumbach 
22580cd58eaaSEmmanuel Grumbach 		if (!block && !(WARN_ON_ONCE(!txq->block))) {
22590cd58eaaSEmmanuel Grumbach 			txq->block--;
22600cd58eaaSEmmanuel Grumbach 			if (!txq->block) {
22610cd58eaaSEmmanuel Grumbach 				iwl_write32(trans, HBUS_TARG_WRPTR,
2262bb98ecd4SSara Sharon 					    txq->write_ptr | (i << 8));
22630cd58eaaSEmmanuel Grumbach 			}
22640cd58eaaSEmmanuel Grumbach 		} else if (block) {
22650cd58eaaSEmmanuel Grumbach 			txq->block++;
22660cd58eaaSEmmanuel Grumbach 		}
22670cd58eaaSEmmanuel Grumbach 
22680cd58eaaSEmmanuel Grumbach 		spin_unlock_bh(&txq->lock);
22690cd58eaaSEmmanuel Grumbach 	}
22700cd58eaaSEmmanuel Grumbach }
22710cd58eaaSEmmanuel Grumbach 
2272e705c121SKalle Valo #define IWL_FLUSH_WAIT_MS	2000
2273e705c121SKalle Valo 
227492536c96SSara Sharon static int iwl_trans_pcie_rxq_dma_data(struct iwl_trans *trans, int queue,
227592536c96SSara Sharon 				       struct iwl_trans_rxq_dma_data *data)
227692536c96SSara Sharon {
227792536c96SSara Sharon 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
227892536c96SSara Sharon 
227992536c96SSara Sharon 	if (queue >= trans->num_rx_queues || !trans_pcie->rxq)
228092536c96SSara Sharon 		return -EINVAL;
228192536c96SSara Sharon 
228292536c96SSara Sharon 	data->fr_bd_cb = trans_pcie->rxq[queue].bd_dma;
228392536c96SSara Sharon 	data->urbd_stts_wrptr = trans_pcie->rxq[queue].rb_stts_dma;
228492536c96SSara Sharon 	data->ur_bd_cb = trans_pcie->rxq[queue].used_bd_dma;
228592536c96SSara Sharon 	data->fr_bd_wid = 0;
228692536c96SSara Sharon 
228792536c96SSara Sharon 	return 0;
228892536c96SSara Sharon }
228992536c96SSara Sharon 
2290d6d517b7SSara Sharon static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, int txq_idx)
2291e705c121SKalle Valo {
2292e705c121SKalle Valo 	struct iwl_txq *txq;
2293e705c121SKalle Valo 	unsigned long now = jiffies;
22942ae48edcSSara Sharon 	bool overflow_tx;
2295e705c121SKalle Valo 	u8 wr_ptr;
2296e705c121SKalle Valo 
22972b3fae66SMatt Chen 	/* Make sure the NIC is still alive in the bus */
2298f60c9e59SEmmanuel Grumbach 	if (test_bit(STATUS_TRANS_DEAD, &trans->status))
2299f60c9e59SEmmanuel Grumbach 		return -ENODEV;
23002b3fae66SMatt Chen 
23014f4822b7SMordechay Goodstein 	if (!test_bit(txq_idx, trans->txqs.queue_used))
2302d6d517b7SSara Sharon 		return -EINVAL;
2303e705c121SKalle Valo 
2304d6d517b7SSara Sharon 	IWL_DEBUG_TX_QUEUES(trans, "Emptying queue %d...\n", txq_idx);
23054f4822b7SMordechay Goodstein 	txq = trans->txqs.txq[txq_idx];
23062ae48edcSSara Sharon 
23072ae48edcSSara Sharon 	spin_lock_bh(&txq->lock);
23082ae48edcSSara Sharon 	overflow_tx = txq->overflow_tx ||
23092ae48edcSSara Sharon 		      !skb_queue_empty(&txq->overflow_q);
23102ae48edcSSara Sharon 	spin_unlock_bh(&txq->lock);
23112ae48edcSSara Sharon 
23126aa7de05SMark Rutland 	wr_ptr = READ_ONCE(txq->write_ptr);
2313e705c121SKalle Valo 
23142ae48edcSSara Sharon 	while ((txq->read_ptr != READ_ONCE(txq->write_ptr) ||
23152ae48edcSSara Sharon 		overflow_tx) &&
2316e705c121SKalle Valo 	       !time_after(jiffies,
2317e705c121SKalle Valo 			   now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS))) {
23186aa7de05SMark Rutland 		u8 write_ptr = READ_ONCE(txq->write_ptr);
2319e705c121SKalle Valo 
23202ae48edcSSara Sharon 		/*
23212ae48edcSSara Sharon 		 * If write pointer moved during the wait, warn only
23222ae48edcSSara Sharon 		 * if the TX came from op mode. In case TX came from
23232ae48edcSSara Sharon 		 * trans layer (overflow TX) don't warn.
23242ae48edcSSara Sharon 		 */
23252ae48edcSSara Sharon 		if (WARN_ONCE(wr_ptr != write_ptr && !overflow_tx,
2326e705c121SKalle Valo 			      "WR pointer moved while flushing %d -> %d\n",
2327e705c121SKalle Valo 			      wr_ptr, write_ptr))
2328e705c121SKalle Valo 			return -ETIMEDOUT;
23292ae48edcSSara Sharon 		wr_ptr = write_ptr;
23302ae48edcSSara Sharon 
2331192185d6SJohannes Berg 		usleep_range(1000, 2000);
23322ae48edcSSara Sharon 
23332ae48edcSSara Sharon 		spin_lock_bh(&txq->lock);
23342ae48edcSSara Sharon 		overflow_tx = txq->overflow_tx ||
23352ae48edcSSara Sharon 			      !skb_queue_empty(&txq->overflow_q);
23362ae48edcSSara Sharon 		spin_unlock_bh(&txq->lock);
2337e705c121SKalle Valo 	}
2338e705c121SKalle Valo 
2339bb98ecd4SSara Sharon 	if (txq->read_ptr != txq->write_ptr) {
2340e705c121SKalle Valo 		IWL_ERR(trans,
2341d6d517b7SSara Sharon 			"fail to flush all tx fifo queues Q %d\n", txq_idx);
23420cd1ad2dSMordechay Goodstein 		iwl_txq_log_scd_error(trans, txq);
2343d6d517b7SSara Sharon 		return -ETIMEDOUT;
2344e705c121SKalle Valo 	}
2345e705c121SKalle Valo 
2346d6d517b7SSara Sharon 	IWL_DEBUG_TX_QUEUES(trans, "Queue %d is now empty.\n", txq_idx);
2347d6d517b7SSara Sharon 
2348d6d517b7SSara Sharon 	return 0;
2349d6d517b7SSara Sharon }
2350d6d517b7SSara Sharon 
2351d6d517b7SSara Sharon static int iwl_trans_pcie_wait_txqs_empty(struct iwl_trans *trans, u32 txq_bm)
2352d6d517b7SSara Sharon {
2353d6d517b7SSara Sharon 	int cnt;
2354d6d517b7SSara Sharon 	int ret = 0;
2355d6d517b7SSara Sharon 
2356d6d517b7SSara Sharon 	/* waiting for all the tx frames complete might take a while */
235779b6c8feSLuca Coelho 	for (cnt = 0;
2358286ca8ebSLuca Coelho 	     cnt < trans->trans_cfg->base_params->num_of_queues;
235979b6c8feSLuca Coelho 	     cnt++) {
2360d6d517b7SSara Sharon 
23614f4822b7SMordechay Goodstein 		if (cnt == trans->txqs.cmd.q_id)
2362d6d517b7SSara Sharon 			continue;
23634f4822b7SMordechay Goodstein 		if (!test_bit(cnt, trans->txqs.queue_used))
2364d6d517b7SSara Sharon 			continue;
2365d6d517b7SSara Sharon 		if (!(BIT(cnt) & txq_bm))
2366d6d517b7SSara Sharon 			continue;
2367d6d517b7SSara Sharon 
2368d6d517b7SSara Sharon 		ret = iwl_trans_pcie_wait_txq_empty(trans, cnt);
236938398efbSSara Sharon 		if (ret)
2370d6d517b7SSara Sharon 			break;
2371d6d517b7SSara Sharon 	}
2372e705c121SKalle Valo 
2373e705c121SKalle Valo 	return ret;
2374e705c121SKalle Valo }
2375e705c121SKalle Valo 
2376e705c121SKalle Valo static void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg,
2377e705c121SKalle Valo 					 u32 mask, u32 value)
2378e705c121SKalle Valo {
2379e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2380e705c121SKalle Valo 	unsigned long flags;
2381e705c121SKalle Valo 
2382e705c121SKalle Valo 	spin_lock_irqsave(&trans_pcie->reg_lock, flags);
2383e705c121SKalle Valo 	__iwl_trans_pcie_set_bits_mask(trans, reg, mask, value);
2384e705c121SKalle Valo 	spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
2385e705c121SKalle Valo }
2386e705c121SKalle Valo 
2387e705c121SKalle Valo static const char *get_csr_string(int cmd)
2388e705c121SKalle Valo {
2389e705c121SKalle Valo #define IWL_CMD(x) case x: return #x
2390e705c121SKalle Valo 	switch (cmd) {
2391e705c121SKalle Valo 	IWL_CMD(CSR_HW_IF_CONFIG_REG);
2392e705c121SKalle Valo 	IWL_CMD(CSR_INT_COALESCING);
2393e705c121SKalle Valo 	IWL_CMD(CSR_INT);
2394e705c121SKalle Valo 	IWL_CMD(CSR_INT_MASK);
2395e705c121SKalle Valo 	IWL_CMD(CSR_FH_INT_STATUS);
2396e705c121SKalle Valo 	IWL_CMD(CSR_GPIO_IN);
2397e705c121SKalle Valo 	IWL_CMD(CSR_RESET);
2398e705c121SKalle Valo 	IWL_CMD(CSR_GP_CNTRL);
2399e705c121SKalle Valo 	IWL_CMD(CSR_HW_REV);
2400e705c121SKalle Valo 	IWL_CMD(CSR_EEPROM_REG);
2401e705c121SKalle Valo 	IWL_CMD(CSR_EEPROM_GP);
2402e705c121SKalle Valo 	IWL_CMD(CSR_OTP_GP_REG);
2403e705c121SKalle Valo 	IWL_CMD(CSR_GIO_REG);
2404e705c121SKalle Valo 	IWL_CMD(CSR_GP_UCODE_REG);
2405e705c121SKalle Valo 	IWL_CMD(CSR_GP_DRIVER_REG);
2406e705c121SKalle Valo 	IWL_CMD(CSR_UCODE_DRV_GP1);
2407e705c121SKalle Valo 	IWL_CMD(CSR_UCODE_DRV_GP2);
2408e705c121SKalle Valo 	IWL_CMD(CSR_LED_REG);
2409e705c121SKalle Valo 	IWL_CMD(CSR_DRAM_INT_TBL_REG);
2410e705c121SKalle Valo 	IWL_CMD(CSR_GIO_CHICKEN_BITS);
2411e705c121SKalle Valo 	IWL_CMD(CSR_ANA_PLL_CFG);
2412e705c121SKalle Valo 	IWL_CMD(CSR_HW_REV_WA_REG);
2413e705c121SKalle Valo 	IWL_CMD(CSR_MONITOR_STATUS_REG);
2414e705c121SKalle Valo 	IWL_CMD(CSR_DBG_HPET_MEM_REG);
2415e705c121SKalle Valo 	default:
2416e705c121SKalle Valo 		return "UNKNOWN";
2417e705c121SKalle Valo 	}
2418e705c121SKalle Valo #undef IWL_CMD
2419e705c121SKalle Valo }
2420e705c121SKalle Valo 
2421e705c121SKalle Valo void iwl_pcie_dump_csr(struct iwl_trans *trans)
2422e705c121SKalle Valo {
2423e705c121SKalle Valo 	int i;
2424e705c121SKalle Valo 	static const u32 csr_tbl[] = {
2425e705c121SKalle Valo 		CSR_HW_IF_CONFIG_REG,
2426e705c121SKalle Valo 		CSR_INT_COALESCING,
2427e705c121SKalle Valo 		CSR_INT,
2428e705c121SKalle Valo 		CSR_INT_MASK,
2429e705c121SKalle Valo 		CSR_FH_INT_STATUS,
2430e705c121SKalle Valo 		CSR_GPIO_IN,
2431e705c121SKalle Valo 		CSR_RESET,
2432e705c121SKalle Valo 		CSR_GP_CNTRL,
2433e705c121SKalle Valo 		CSR_HW_REV,
2434e705c121SKalle Valo 		CSR_EEPROM_REG,
2435e705c121SKalle Valo 		CSR_EEPROM_GP,
2436e705c121SKalle Valo 		CSR_OTP_GP_REG,
2437e705c121SKalle Valo 		CSR_GIO_REG,
2438e705c121SKalle Valo 		CSR_GP_UCODE_REG,
2439e705c121SKalle Valo 		CSR_GP_DRIVER_REG,
2440e705c121SKalle Valo 		CSR_UCODE_DRV_GP1,
2441e705c121SKalle Valo 		CSR_UCODE_DRV_GP2,
2442e705c121SKalle Valo 		CSR_LED_REG,
2443e705c121SKalle Valo 		CSR_DRAM_INT_TBL_REG,
2444e705c121SKalle Valo 		CSR_GIO_CHICKEN_BITS,
2445e705c121SKalle Valo 		CSR_ANA_PLL_CFG,
2446e705c121SKalle Valo 		CSR_MONITOR_STATUS_REG,
2447e705c121SKalle Valo 		CSR_HW_REV_WA_REG,
2448e705c121SKalle Valo 		CSR_DBG_HPET_MEM_REG
2449e705c121SKalle Valo 	};
2450e705c121SKalle Valo 	IWL_ERR(trans, "CSR values:\n");
2451e705c121SKalle Valo 	IWL_ERR(trans, "(2nd byte of CSR_INT_COALESCING is "
2452e705c121SKalle Valo 		"CSR_INT_PERIODIC_REG)\n");
2453e705c121SKalle Valo 	for (i = 0; i <  ARRAY_SIZE(csr_tbl); i++) {
2454e705c121SKalle Valo 		IWL_ERR(trans, "  %25s: 0X%08x\n",
2455e705c121SKalle Valo 			get_csr_string(csr_tbl[i]),
2456e705c121SKalle Valo 			iwl_read32(trans, csr_tbl[i]));
2457e705c121SKalle Valo 	}
2458e705c121SKalle Valo }
2459e705c121SKalle Valo 
2460e705c121SKalle Valo #ifdef CONFIG_IWLWIFI_DEBUGFS
2461e705c121SKalle Valo /* create and remove of files */
2462e705c121SKalle Valo #define DEBUGFS_ADD_FILE(name, parent, mode) do {			\
2463cf5d5663SGreg Kroah-Hartman 	debugfs_create_file(#name, mode, parent, trans,			\
2464cf5d5663SGreg Kroah-Hartman 			    &iwl_dbgfs_##name##_ops);			\
2465e705c121SKalle Valo } while (0)
2466e705c121SKalle Valo 
2467e705c121SKalle Valo /* file operation */
2468e705c121SKalle Valo #define DEBUGFS_READ_FILE_OPS(name)					\
2469e705c121SKalle Valo static const struct file_operations iwl_dbgfs_##name##_ops = {		\
2470e705c121SKalle Valo 	.read = iwl_dbgfs_##name##_read,				\
2471e705c121SKalle Valo 	.open = simple_open,						\
2472e705c121SKalle Valo 	.llseek = generic_file_llseek,					\
2473e705c121SKalle Valo };
2474e705c121SKalle Valo 
2475e705c121SKalle Valo #define DEBUGFS_WRITE_FILE_OPS(name)                                    \
2476e705c121SKalle Valo static const struct file_operations iwl_dbgfs_##name##_ops = {          \
2477e705c121SKalle Valo 	.write = iwl_dbgfs_##name##_write,                              \
2478e705c121SKalle Valo 	.open = simple_open,						\
2479e705c121SKalle Valo 	.llseek = generic_file_llseek,					\
2480e705c121SKalle Valo };
2481e705c121SKalle Valo 
2482e705c121SKalle Valo #define DEBUGFS_READ_WRITE_FILE_OPS(name)				\
2483e705c121SKalle Valo static const struct file_operations iwl_dbgfs_##name##_ops = {		\
2484e705c121SKalle Valo 	.write = iwl_dbgfs_##name##_write,				\
2485e705c121SKalle Valo 	.read = iwl_dbgfs_##name##_read,				\
2486e705c121SKalle Valo 	.open = simple_open,						\
2487e705c121SKalle Valo 	.llseek = generic_file_llseek,					\
2488e705c121SKalle Valo };
2489e705c121SKalle Valo 
2490df67a1beSJohannes Berg struct iwl_dbgfs_tx_queue_priv {
2491df67a1beSJohannes Berg 	struct iwl_trans *trans;
2492df67a1beSJohannes Berg };
2493df67a1beSJohannes Berg 
2494df67a1beSJohannes Berg struct iwl_dbgfs_tx_queue_state {
2495df67a1beSJohannes Berg 	loff_t pos;
2496df67a1beSJohannes Berg };
2497df67a1beSJohannes Berg 
2498df67a1beSJohannes Berg static void *iwl_dbgfs_tx_queue_seq_start(struct seq_file *seq, loff_t *pos)
2499e705c121SKalle Valo {
2500df67a1beSJohannes Berg 	struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
2501df67a1beSJohannes Berg 	struct iwl_dbgfs_tx_queue_state *state;
2502df67a1beSJohannes Berg 
2503df67a1beSJohannes Berg 	if (*pos >= priv->trans->trans_cfg->base_params->num_of_queues)
2504df67a1beSJohannes Berg 		return NULL;
2505df67a1beSJohannes Berg 
2506df67a1beSJohannes Berg 	state = kmalloc(sizeof(*state), GFP_KERNEL);
2507df67a1beSJohannes Berg 	if (!state)
2508df67a1beSJohannes Berg 		return NULL;
2509df67a1beSJohannes Berg 	state->pos = *pos;
2510df67a1beSJohannes Berg 	return state;
2511df67a1beSJohannes Berg }
2512df67a1beSJohannes Berg 
2513df67a1beSJohannes Berg static void *iwl_dbgfs_tx_queue_seq_next(struct seq_file *seq,
2514df67a1beSJohannes Berg 					 void *v, loff_t *pos)
2515df67a1beSJohannes Berg {
2516df67a1beSJohannes Berg 	struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
2517df67a1beSJohannes Berg 	struct iwl_dbgfs_tx_queue_state *state = v;
2518df67a1beSJohannes Berg 
2519df67a1beSJohannes Berg 	*pos = ++state->pos;
2520df67a1beSJohannes Berg 
2521df67a1beSJohannes Berg 	if (*pos >= priv->trans->trans_cfg->base_params->num_of_queues)
2522df67a1beSJohannes Berg 		return NULL;
2523df67a1beSJohannes Berg 
2524df67a1beSJohannes Berg 	return state;
2525df67a1beSJohannes Berg }
2526df67a1beSJohannes Berg 
2527df67a1beSJohannes Berg static void iwl_dbgfs_tx_queue_seq_stop(struct seq_file *seq, void *v)
2528df67a1beSJohannes Berg {
2529df67a1beSJohannes Berg 	kfree(v);
2530df67a1beSJohannes Berg }
2531df67a1beSJohannes Berg 
2532df67a1beSJohannes Berg static int iwl_dbgfs_tx_queue_seq_show(struct seq_file *seq, void *v)
2533df67a1beSJohannes Berg {
2534df67a1beSJohannes Berg 	struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
2535df67a1beSJohannes Berg 	struct iwl_dbgfs_tx_queue_state *state = v;
2536df67a1beSJohannes Berg 	struct iwl_trans *trans = priv->trans;
25374f4822b7SMordechay Goodstein 	struct iwl_txq *txq = trans->txqs.txq[state->pos];
2538e705c121SKalle Valo 
2539df67a1beSJohannes Berg 	seq_printf(seq, "hwq %.3u: used=%d stopped=%d ",
2540df67a1beSJohannes Berg 		   (unsigned int)state->pos,
25414f4822b7SMordechay Goodstein 		   !!test_bit(state->pos, trans->txqs.queue_used),
25424f4822b7SMordechay Goodstein 		   !!test_bit(state->pos, trans->txqs.queue_stopped));
2543df67a1beSJohannes Berg 	if (txq)
2544df67a1beSJohannes Berg 		seq_printf(seq,
254595a9e44fSJohannes Berg 			   "read=%u write=%u need_update=%d frozen=%d n_window=%d ampdu=%d",
2546df67a1beSJohannes Berg 			   txq->read_ptr, txq->write_ptr,
254795a9e44fSJohannes Berg 			   txq->need_update, txq->frozen,
254895a9e44fSJohannes Berg 			   txq->n_window, txq->ampdu);
2549df67a1beSJohannes Berg 	else
2550df67a1beSJohannes Berg 		seq_puts(seq, "(unallocated)");
2551e705c121SKalle Valo 
25524f4822b7SMordechay Goodstein 	if (state->pos == trans->txqs.cmd.q_id)
2553df67a1beSJohannes Berg 		seq_puts(seq, " (HCMD)");
2554df67a1beSJohannes Berg 	seq_puts(seq, "\n");
2555e705c121SKalle Valo 
2556df67a1beSJohannes Berg 	return 0;
2557df67a1beSJohannes Berg }
2558df67a1beSJohannes Berg 
2559df67a1beSJohannes Berg static const struct seq_operations iwl_dbgfs_tx_queue_seq_ops = {
2560df67a1beSJohannes Berg 	.start = iwl_dbgfs_tx_queue_seq_start,
2561df67a1beSJohannes Berg 	.next = iwl_dbgfs_tx_queue_seq_next,
2562df67a1beSJohannes Berg 	.stop = iwl_dbgfs_tx_queue_seq_stop,
2563df67a1beSJohannes Berg 	.show = iwl_dbgfs_tx_queue_seq_show,
2564df67a1beSJohannes Berg };
2565df67a1beSJohannes Berg 
2566df67a1beSJohannes Berg static int iwl_dbgfs_tx_queue_open(struct inode *inode, struct file *filp)
2567df67a1beSJohannes Berg {
2568df67a1beSJohannes Berg 	struct iwl_dbgfs_tx_queue_priv *priv;
2569df67a1beSJohannes Berg 
2570df67a1beSJohannes Berg 	priv = __seq_open_private(filp, &iwl_dbgfs_tx_queue_seq_ops,
2571df67a1beSJohannes Berg 				  sizeof(*priv));
2572df67a1beSJohannes Berg 
2573df67a1beSJohannes Berg 	if (!priv)
2574e705c121SKalle Valo 		return -ENOMEM;
2575e705c121SKalle Valo 
2576df67a1beSJohannes Berg 	priv->trans = inode->i_private;
2577df67a1beSJohannes Berg 	return 0;
2578e705c121SKalle Valo }
2579e705c121SKalle Valo 
2580e705c121SKalle Valo static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
2581e705c121SKalle Valo 				       char __user *user_buf,
2582e705c121SKalle Valo 				       size_t count, loff_t *ppos)
2583e705c121SKalle Valo {
2584e705c121SKalle Valo 	struct iwl_trans *trans = file->private_data;
2585e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
258678485054SSara Sharon 	char *buf;
258778485054SSara Sharon 	int pos = 0, i, ret;
2588eb3dc36eSColin Ian King 	size_t bufsz;
2589e705c121SKalle Valo 
259078485054SSara Sharon 	bufsz = sizeof(char) * 121 * trans->num_rx_queues;
259178485054SSara Sharon 
259278485054SSara Sharon 	if (!trans_pcie->rxq)
259378485054SSara Sharon 		return -EAGAIN;
259478485054SSara Sharon 
259578485054SSara Sharon 	buf = kzalloc(bufsz, GFP_KERNEL);
259678485054SSara Sharon 	if (!buf)
259778485054SSara Sharon 		return -ENOMEM;
259878485054SSara Sharon 
259978485054SSara Sharon 	for (i = 0; i < trans->num_rx_queues && pos < bufsz; i++) {
260078485054SSara Sharon 		struct iwl_rxq *rxq = &trans_pcie->rxq[i];
260178485054SSara Sharon 
260278485054SSara Sharon 		pos += scnprintf(buf + pos, bufsz - pos, "queue#: %2d\n",
260378485054SSara Sharon 				 i);
260478485054SSara Sharon 		pos += scnprintf(buf + pos, bufsz - pos, "\tread: %u\n",
2605e705c121SKalle Valo 				 rxq->read);
260678485054SSara Sharon 		pos += scnprintf(buf + pos, bufsz - pos, "\twrite: %u\n",
2607e705c121SKalle Valo 				 rxq->write);
260878485054SSara Sharon 		pos += scnprintf(buf + pos, bufsz - pos, "\twrite_actual: %u\n",
2609e705c121SKalle Valo 				 rxq->write_actual);
261078485054SSara Sharon 		pos += scnprintf(buf + pos, bufsz - pos, "\tneed_update: %2d\n",
2611e705c121SKalle Valo 				 rxq->need_update);
261278485054SSara Sharon 		pos += scnprintf(buf + pos, bufsz - pos, "\tfree_count: %u\n",
2613e705c121SKalle Valo 				 rxq->free_count);
2614e705c121SKalle Valo 		if (rxq->rb_stts) {
26150307c839SGolan Ben Ami 			u32 r =	__le16_to_cpu(iwl_get_closed_rb_stts(trans,
26160307c839SGolan Ben Ami 								     rxq));
261778485054SSara Sharon 			pos += scnprintf(buf + pos, bufsz - pos,
261878485054SSara Sharon 					 "\tclosed_rb_num: %u\n",
26190307c839SGolan Ben Ami 					 r & 0x0FFF);
2620e705c121SKalle Valo 		} else {
2621e705c121SKalle Valo 			pos += scnprintf(buf + pos, bufsz - pos,
262278485054SSara Sharon 					 "\tclosed_rb_num: Not Allocated\n");
2623e705c121SKalle Valo 		}
262478485054SSara Sharon 	}
262578485054SSara Sharon 	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
262678485054SSara Sharon 	kfree(buf);
262778485054SSara Sharon 
262878485054SSara Sharon 	return ret;
2629e705c121SKalle Valo }
2630e705c121SKalle Valo 
2631e705c121SKalle Valo static ssize_t iwl_dbgfs_interrupt_read(struct file *file,
2632e705c121SKalle Valo 					char __user *user_buf,
2633e705c121SKalle Valo 					size_t count, loff_t *ppos)
2634e705c121SKalle Valo {
2635e705c121SKalle Valo 	struct iwl_trans *trans = file->private_data;
2636e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2637e705c121SKalle Valo 	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
2638e705c121SKalle Valo 
2639e705c121SKalle Valo 	int pos = 0;
2640e705c121SKalle Valo 	char *buf;
2641e705c121SKalle Valo 	int bufsz = 24 * 64; /* 24 items * 64 char per item */
2642e705c121SKalle Valo 	ssize_t ret;
2643e705c121SKalle Valo 
2644e705c121SKalle Valo 	buf = kzalloc(bufsz, GFP_KERNEL);
2645e705c121SKalle Valo 	if (!buf)
2646e705c121SKalle Valo 		return -ENOMEM;
2647e705c121SKalle Valo 
2648e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos,
2649e705c121SKalle Valo 			"Interrupt Statistics Report:\n");
2650e705c121SKalle Valo 
2651e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos, "HW Error:\t\t\t %u\n",
2652e705c121SKalle Valo 		isr_stats->hw);
2653e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos, "SW Error:\t\t\t %u\n",
2654e705c121SKalle Valo 		isr_stats->sw);
2655e705c121SKalle Valo 	if (isr_stats->sw || isr_stats->hw) {
2656e705c121SKalle Valo 		pos += scnprintf(buf + pos, bufsz - pos,
2657e705c121SKalle Valo 			"\tLast Restarting Code:  0x%X\n",
2658e705c121SKalle Valo 			isr_stats->err_code);
2659e705c121SKalle Valo 	}
2660e705c121SKalle Valo #ifdef CONFIG_IWLWIFI_DEBUG
2661e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos, "Frame transmitted:\t\t %u\n",
2662e705c121SKalle Valo 		isr_stats->sch);
2663e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos, "Alive interrupt:\t\t %u\n",
2664e705c121SKalle Valo 		isr_stats->alive);
2665e705c121SKalle Valo #endif
2666e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos,
2667e705c121SKalle Valo 		"HW RF KILL switch toggled:\t %u\n", isr_stats->rfkill);
2668e705c121SKalle Valo 
2669e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos, "CT KILL:\t\t\t %u\n",
2670e705c121SKalle Valo 		isr_stats->ctkill);
2671e705c121SKalle Valo 
2672e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos, "Wakeup Interrupt:\t\t %u\n",
2673e705c121SKalle Valo 		isr_stats->wakeup);
2674e705c121SKalle Valo 
2675e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos,
2676e705c121SKalle Valo 		"Rx command responses:\t\t %u\n", isr_stats->rx);
2677e705c121SKalle Valo 
2678e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos, "Tx/FH interrupt:\t\t %u\n",
2679e705c121SKalle Valo 		isr_stats->tx);
2680e705c121SKalle Valo 
2681e705c121SKalle Valo 	pos += scnprintf(buf + pos, bufsz - pos, "Unexpected INTA:\t\t %u\n",
2682e705c121SKalle Valo 		isr_stats->unhandled);
2683e705c121SKalle Valo 
2684e705c121SKalle Valo 	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2685e705c121SKalle Valo 	kfree(buf);
2686e705c121SKalle Valo 	return ret;
2687e705c121SKalle Valo }
2688e705c121SKalle Valo 
2689e705c121SKalle Valo static ssize_t iwl_dbgfs_interrupt_write(struct file *file,
2690e705c121SKalle Valo 					 const char __user *user_buf,
2691e705c121SKalle Valo 					 size_t count, loff_t *ppos)
2692e705c121SKalle Valo {
2693e705c121SKalle Valo 	struct iwl_trans *trans = file->private_data;
2694e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2695e705c121SKalle Valo 	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
2696e705c121SKalle Valo 	u32 reset_flag;
2697078f1131SJohannes Berg 	int ret;
2698e705c121SKalle Valo 
2699078f1131SJohannes Berg 	ret = kstrtou32_from_user(user_buf, count, 16, &reset_flag);
2700078f1131SJohannes Berg 	if (ret)
2701078f1131SJohannes Berg 		return ret;
2702e705c121SKalle Valo 	if (reset_flag == 0)
2703e705c121SKalle Valo 		memset(isr_stats, 0, sizeof(*isr_stats));
2704e705c121SKalle Valo 
2705e705c121SKalle Valo 	return count;
2706e705c121SKalle Valo }
2707e705c121SKalle Valo 
2708e705c121SKalle Valo static ssize_t iwl_dbgfs_csr_write(struct file *file,
2709e705c121SKalle Valo 				   const char __user *user_buf,
2710e705c121SKalle Valo 				   size_t count, loff_t *ppos)
2711e705c121SKalle Valo {
2712e705c121SKalle Valo 	struct iwl_trans *trans = file->private_data;
2713e705c121SKalle Valo 
2714e705c121SKalle Valo 	iwl_pcie_dump_csr(trans);
2715e705c121SKalle Valo 
2716e705c121SKalle Valo 	return count;
2717e705c121SKalle Valo }
2718e705c121SKalle Valo 
2719e705c121SKalle Valo static ssize_t iwl_dbgfs_fh_reg_read(struct file *file,
2720e705c121SKalle Valo 				     char __user *user_buf,
2721e705c121SKalle Valo 				     size_t count, loff_t *ppos)
2722e705c121SKalle Valo {
2723e705c121SKalle Valo 	struct iwl_trans *trans = file->private_data;
2724e705c121SKalle Valo 	char *buf = NULL;
2725e705c121SKalle Valo 	ssize_t ret;
2726e705c121SKalle Valo 
2727e705c121SKalle Valo 	ret = iwl_dump_fh(trans, &buf);
2728e705c121SKalle Valo 	if (ret < 0)
2729e705c121SKalle Valo 		return ret;
2730e705c121SKalle Valo 	if (!buf)
2731e705c121SKalle Valo 		return -EINVAL;
2732e705c121SKalle Valo 	ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret);
2733e705c121SKalle Valo 	kfree(buf);
2734e705c121SKalle Valo 	return ret;
2735e705c121SKalle Valo }
2736e705c121SKalle Valo 
2737fa4de7f7SJohannes Berg static ssize_t iwl_dbgfs_rfkill_read(struct file *file,
2738fa4de7f7SJohannes Berg 				     char __user *user_buf,
2739fa4de7f7SJohannes Berg 				     size_t count, loff_t *ppos)
2740fa4de7f7SJohannes Berg {
2741fa4de7f7SJohannes Berg 	struct iwl_trans *trans = file->private_data;
2742fa4de7f7SJohannes Berg 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2743fa4de7f7SJohannes Berg 	char buf[100];
2744fa4de7f7SJohannes Berg 	int pos;
2745fa4de7f7SJohannes Berg 
2746fa4de7f7SJohannes Berg 	pos = scnprintf(buf, sizeof(buf), "debug: %d\nhw: %d\n",
2747fa4de7f7SJohannes Berg 			trans_pcie->debug_rfkill,
2748fa4de7f7SJohannes Berg 			!(iwl_read32(trans, CSR_GP_CNTRL) &
2749fa4de7f7SJohannes Berg 				CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW));
2750fa4de7f7SJohannes Berg 
2751fa4de7f7SJohannes Berg 	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2752fa4de7f7SJohannes Berg }
2753fa4de7f7SJohannes Berg 
2754fa4de7f7SJohannes Berg static ssize_t iwl_dbgfs_rfkill_write(struct file *file,
2755fa4de7f7SJohannes Berg 				      const char __user *user_buf,
2756fa4de7f7SJohannes Berg 				      size_t count, loff_t *ppos)
2757fa4de7f7SJohannes Berg {
2758fa4de7f7SJohannes Berg 	struct iwl_trans *trans = file->private_data;
2759fa4de7f7SJohannes Berg 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2760c5bf4fa1SJohannes Berg 	bool new_value;
2761fa4de7f7SJohannes Berg 	int ret;
2762fa4de7f7SJohannes Berg 
2763c5bf4fa1SJohannes Berg 	ret = kstrtobool_from_user(user_buf, count, &new_value);
2764fa4de7f7SJohannes Berg 	if (ret)
2765fa4de7f7SJohannes Berg 		return ret;
2766c5bf4fa1SJohannes Berg 	if (new_value == trans_pcie->debug_rfkill)
2767fa4de7f7SJohannes Berg 		return count;
2768fa4de7f7SJohannes Berg 	IWL_WARN(trans, "changing debug rfkill %d->%d\n",
2769c5bf4fa1SJohannes Berg 		 trans_pcie->debug_rfkill, new_value);
2770c5bf4fa1SJohannes Berg 	trans_pcie->debug_rfkill = new_value;
2771fa4de7f7SJohannes Berg 	iwl_pcie_handle_rfkill_irq(trans);
2772fa4de7f7SJohannes Berg 
2773fa4de7f7SJohannes Berg 	return count;
2774fa4de7f7SJohannes Berg }
2775fa4de7f7SJohannes Berg 
2776f7805b33SLior Cohen static int iwl_dbgfs_monitor_data_open(struct inode *inode,
2777f7805b33SLior Cohen 				       struct file *file)
2778f7805b33SLior Cohen {
2779f7805b33SLior Cohen 	struct iwl_trans *trans = inode->i_private;
2780f7805b33SLior Cohen 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2781f7805b33SLior Cohen 
278291c28b83SShahar S Matityahu 	if (!trans->dbg.dest_tlv ||
278391c28b83SShahar S Matityahu 	    trans->dbg.dest_tlv->monitor_mode != EXTERNAL_MODE) {
2784f7805b33SLior Cohen 		IWL_ERR(trans, "Debug destination is not set to DRAM\n");
2785f7805b33SLior Cohen 		return -ENOENT;
2786f7805b33SLior Cohen 	}
2787f7805b33SLior Cohen 
2788f7805b33SLior Cohen 	if (trans_pcie->fw_mon_data.state != IWL_FW_MON_DBGFS_STATE_CLOSED)
2789f7805b33SLior Cohen 		return -EBUSY;
2790f7805b33SLior Cohen 
2791f7805b33SLior Cohen 	trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_OPEN;
2792f7805b33SLior Cohen 	return simple_open(inode, file);
2793f7805b33SLior Cohen }
2794f7805b33SLior Cohen 
2795f7805b33SLior Cohen static int iwl_dbgfs_monitor_data_release(struct inode *inode,
2796f7805b33SLior Cohen 					  struct file *file)
2797f7805b33SLior Cohen {
2798f7805b33SLior Cohen 	struct iwl_trans_pcie *trans_pcie =
2799f7805b33SLior Cohen 		IWL_TRANS_GET_PCIE_TRANS(inode->i_private);
2800f7805b33SLior Cohen 
2801f7805b33SLior Cohen 	if (trans_pcie->fw_mon_data.state == IWL_FW_MON_DBGFS_STATE_OPEN)
2802f7805b33SLior Cohen 		trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED;
2803f7805b33SLior Cohen 	return 0;
2804f7805b33SLior Cohen }
2805f7805b33SLior Cohen 
2806f7805b33SLior Cohen static bool iwl_write_to_user_buf(char __user *user_buf, ssize_t count,
2807f7805b33SLior Cohen 				  void *buf, ssize_t *size,
2808f7805b33SLior Cohen 				  ssize_t *bytes_copied)
2809f7805b33SLior Cohen {
2810f7805b33SLior Cohen 	int buf_size_left = count - *bytes_copied;
2811f7805b33SLior Cohen 
2812f7805b33SLior Cohen 	buf_size_left = buf_size_left - (buf_size_left % sizeof(u32));
2813f7805b33SLior Cohen 	if (*size > buf_size_left)
2814f7805b33SLior Cohen 		*size = buf_size_left;
2815f7805b33SLior Cohen 
2816f7805b33SLior Cohen 	*size -= copy_to_user(user_buf, buf, *size);
2817f7805b33SLior Cohen 	*bytes_copied += *size;
2818f7805b33SLior Cohen 
2819f7805b33SLior Cohen 	if (buf_size_left == *size)
2820f7805b33SLior Cohen 		return true;
2821f7805b33SLior Cohen 	return false;
2822f7805b33SLior Cohen }
2823f7805b33SLior Cohen 
2824f7805b33SLior Cohen static ssize_t iwl_dbgfs_monitor_data_read(struct file *file,
2825f7805b33SLior Cohen 					   char __user *user_buf,
2826f7805b33SLior Cohen 					   size_t count, loff_t *ppos)
2827f7805b33SLior Cohen {
2828f7805b33SLior Cohen 	struct iwl_trans *trans = file->private_data;
2829f7805b33SLior Cohen 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
283069f0e505SShahar S Matityahu 	void *cpu_addr = (void *)trans->dbg.fw_mon.block, *curr_buf;
2831f7805b33SLior Cohen 	struct cont_rec *data = &trans_pcie->fw_mon_data;
2832f7805b33SLior Cohen 	u32 write_ptr_addr, wrap_cnt_addr, write_ptr, wrap_cnt;
2833f7805b33SLior Cohen 	ssize_t size, bytes_copied = 0;
2834f7805b33SLior Cohen 	bool b_full;
2835f7805b33SLior Cohen 
283691c28b83SShahar S Matityahu 	if (trans->dbg.dest_tlv) {
2837f7805b33SLior Cohen 		write_ptr_addr =
283891c28b83SShahar S Matityahu 			le32_to_cpu(trans->dbg.dest_tlv->write_ptr_reg);
283991c28b83SShahar S Matityahu 		wrap_cnt_addr = le32_to_cpu(trans->dbg.dest_tlv->wrap_count);
2840f7805b33SLior Cohen 	} else {
2841f7805b33SLior Cohen 		write_ptr_addr = MON_BUFF_WRPTR;
2842f7805b33SLior Cohen 		wrap_cnt_addr = MON_BUFF_CYCLE_CNT;
2843f7805b33SLior Cohen 	}
2844f7805b33SLior Cohen 
284591c28b83SShahar S Matityahu 	if (unlikely(!trans->dbg.rec_on))
2846f7805b33SLior Cohen 		return 0;
2847f7805b33SLior Cohen 
2848f7805b33SLior Cohen 	mutex_lock(&data->mutex);
2849f7805b33SLior Cohen 	if (data->state ==
2850f7805b33SLior Cohen 	    IWL_FW_MON_DBGFS_STATE_DISABLED) {
2851f7805b33SLior Cohen 		mutex_unlock(&data->mutex);
2852f7805b33SLior Cohen 		return 0;
2853f7805b33SLior Cohen 	}
2854f7805b33SLior Cohen 
2855f7805b33SLior Cohen 	/* write_ptr position in bytes rather then DW */
2856f7805b33SLior Cohen 	write_ptr = iwl_read_prph(trans, write_ptr_addr) * sizeof(u32);
2857f7805b33SLior Cohen 	wrap_cnt = iwl_read_prph(trans, wrap_cnt_addr);
2858f7805b33SLior Cohen 
2859f7805b33SLior Cohen 	if (data->prev_wrap_cnt == wrap_cnt) {
2860f7805b33SLior Cohen 		size = write_ptr - data->prev_wr_ptr;
2861f7805b33SLior Cohen 		curr_buf = cpu_addr + data->prev_wr_ptr;
2862f7805b33SLior Cohen 		b_full = iwl_write_to_user_buf(user_buf, count,
2863f7805b33SLior Cohen 					       curr_buf, &size,
2864f7805b33SLior Cohen 					       &bytes_copied);
2865f7805b33SLior Cohen 		data->prev_wr_ptr += size;
2866f7805b33SLior Cohen 
2867f7805b33SLior Cohen 	} else if (data->prev_wrap_cnt == wrap_cnt - 1 &&
2868f7805b33SLior Cohen 		   write_ptr < data->prev_wr_ptr) {
286969f0e505SShahar S Matityahu 		size = trans->dbg.fw_mon.size - data->prev_wr_ptr;
2870f7805b33SLior Cohen 		curr_buf = cpu_addr + data->prev_wr_ptr;
2871f7805b33SLior Cohen 		b_full = iwl_write_to_user_buf(user_buf, count,
2872f7805b33SLior Cohen 					       curr_buf, &size,
2873f7805b33SLior Cohen 					       &bytes_copied);
2874f7805b33SLior Cohen 		data->prev_wr_ptr += size;
2875f7805b33SLior Cohen 
2876f7805b33SLior Cohen 		if (!b_full) {
2877f7805b33SLior Cohen 			size = write_ptr;
2878f7805b33SLior Cohen 			b_full = iwl_write_to_user_buf(user_buf, count,
2879f7805b33SLior Cohen 						       cpu_addr, &size,
2880f7805b33SLior Cohen 						       &bytes_copied);
2881f7805b33SLior Cohen 			data->prev_wr_ptr = size;
2882f7805b33SLior Cohen 			data->prev_wrap_cnt++;
2883f7805b33SLior Cohen 		}
2884f7805b33SLior Cohen 	} else {
2885f7805b33SLior Cohen 		if (data->prev_wrap_cnt == wrap_cnt - 1 &&
2886f7805b33SLior Cohen 		    write_ptr > data->prev_wr_ptr)
2887f7805b33SLior Cohen 			IWL_WARN(trans,
2888f7805b33SLior Cohen 				 "write pointer passed previous write pointer, start copying from the beginning\n");
2889f7805b33SLior Cohen 		else if (!unlikely(data->prev_wrap_cnt == 0 &&
2890f7805b33SLior Cohen 				   data->prev_wr_ptr == 0))
2891f7805b33SLior Cohen 			IWL_WARN(trans,
2892f7805b33SLior Cohen 				 "monitor data is out of sync, start copying from the beginning\n");
2893f7805b33SLior Cohen 
2894f7805b33SLior Cohen 		size = write_ptr;
2895f7805b33SLior Cohen 		b_full = iwl_write_to_user_buf(user_buf, count,
2896f7805b33SLior Cohen 					       cpu_addr, &size,
2897f7805b33SLior Cohen 					       &bytes_copied);
2898f7805b33SLior Cohen 		data->prev_wr_ptr = size;
2899f7805b33SLior Cohen 		data->prev_wrap_cnt = wrap_cnt;
2900f7805b33SLior Cohen 	}
2901f7805b33SLior Cohen 
2902f7805b33SLior Cohen 	mutex_unlock(&data->mutex);
2903f7805b33SLior Cohen 
2904f7805b33SLior Cohen 	return bytes_copied;
2905f7805b33SLior Cohen }
2906f7805b33SLior Cohen 
2907e705c121SKalle Valo DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
2908e705c121SKalle Valo DEBUGFS_READ_FILE_OPS(fh_reg);
2909e705c121SKalle Valo DEBUGFS_READ_FILE_OPS(rx_queue);
2910e705c121SKalle Valo DEBUGFS_WRITE_FILE_OPS(csr);
2911fa4de7f7SJohannes Berg DEBUGFS_READ_WRITE_FILE_OPS(rfkill);
2912df67a1beSJohannes Berg static const struct file_operations iwl_dbgfs_tx_queue_ops = {
2913df67a1beSJohannes Berg 	.owner = THIS_MODULE,
2914df67a1beSJohannes Berg 	.open = iwl_dbgfs_tx_queue_open,
2915df67a1beSJohannes Berg 	.read = seq_read,
2916df67a1beSJohannes Berg 	.llseek = seq_lseek,
2917df67a1beSJohannes Berg 	.release = seq_release_private,
2918df67a1beSJohannes Berg };
2919e705c121SKalle Valo 
2920f7805b33SLior Cohen static const struct file_operations iwl_dbgfs_monitor_data_ops = {
2921f7805b33SLior Cohen 	.read = iwl_dbgfs_monitor_data_read,
2922f7805b33SLior Cohen 	.open = iwl_dbgfs_monitor_data_open,
2923f7805b33SLior Cohen 	.release = iwl_dbgfs_monitor_data_release,
2924f7805b33SLior Cohen };
2925f7805b33SLior Cohen 
2926f8a1edb7SJohannes Berg /* Create the debugfs files and directories */
2927cf5d5663SGreg Kroah-Hartman void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
2928e705c121SKalle Valo {
2929f8a1edb7SJohannes Berg 	struct dentry *dir = trans->dbgfs_dir;
2930f8a1edb7SJohannes Berg 
29312ef00c53SJoe Perches 	DEBUGFS_ADD_FILE(rx_queue, dir, 0400);
29322ef00c53SJoe Perches 	DEBUGFS_ADD_FILE(tx_queue, dir, 0400);
29332ef00c53SJoe Perches 	DEBUGFS_ADD_FILE(interrupt, dir, 0600);
29342ef00c53SJoe Perches 	DEBUGFS_ADD_FILE(csr, dir, 0200);
29352ef00c53SJoe Perches 	DEBUGFS_ADD_FILE(fh_reg, dir, 0400);
29362ef00c53SJoe Perches 	DEBUGFS_ADD_FILE(rfkill, dir, 0600);
2937f7805b33SLior Cohen 	DEBUGFS_ADD_FILE(monitor_data, dir, 0400);
2938e705c121SKalle Valo }
2939f7805b33SLior Cohen 
2940f7805b33SLior Cohen static void iwl_trans_pcie_debugfs_cleanup(struct iwl_trans *trans)
2941f7805b33SLior Cohen {
2942f7805b33SLior Cohen 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2943f7805b33SLior Cohen 	struct cont_rec *data = &trans_pcie->fw_mon_data;
2944f7805b33SLior Cohen 
2945f7805b33SLior Cohen 	mutex_lock(&data->mutex);
2946f7805b33SLior Cohen 	data->state = IWL_FW_MON_DBGFS_STATE_DISABLED;
2947f7805b33SLior Cohen 	mutex_unlock(&data->mutex);
2948f7805b33SLior Cohen }
2949e705c121SKalle Valo #endif /*CONFIG_IWLWIFI_DEBUGFS */
2950e705c121SKalle Valo 
29516983ba69SSara Sharon static u32 iwl_trans_pcie_get_cmdlen(struct iwl_trans *trans, void *tfd)
2952e705c121SKalle Valo {
2953e705c121SKalle Valo 	u32 cmdlen = 0;
2954e705c121SKalle Valo 	int i;
2955e705c121SKalle Valo 
2956885375d0SMordechay Goodstein 	for (i = 0; i < trans->txqs.tfd.max_tbs; i++)
29570179bfffSMordechay Goodstein 		cmdlen += iwl_txq_gen1_tfd_tb_get_len(trans, tfd, i);
2958e705c121SKalle Valo 
2959e705c121SKalle Valo 	return cmdlen;
2960e705c121SKalle Valo }
2961e705c121SKalle Valo 
2962e705c121SKalle Valo static u32 iwl_trans_pcie_dump_rbs(struct iwl_trans *trans,
2963e705c121SKalle Valo 				   struct iwl_fw_error_dump_data **data,
2964e705c121SKalle Valo 				   int allocated_rb_nums)
2965e705c121SKalle Valo {
2966e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
296780084e35SJohannes Berg 	int max_len = trans_pcie->rx_buf_bytes;
296878485054SSara Sharon 	/* Dump RBs is supported only for pre-9000 devices (1 queue) */
296978485054SSara Sharon 	struct iwl_rxq *rxq = &trans_pcie->rxq[0];
2970e705c121SKalle Valo 	u32 i, r, j, rb_len = 0;
2971e705c121SKalle Valo 
2972e705c121SKalle Valo 	spin_lock(&rxq->lock);
2973e705c121SKalle Valo 
29740307c839SGolan Ben Ami 	r = le16_to_cpu(iwl_get_closed_rb_stts(trans, rxq)) & 0x0FFF;
2975e705c121SKalle Valo 
2976e705c121SKalle Valo 	for (i = rxq->read, j = 0;
2977e705c121SKalle Valo 	     i != r && j < allocated_rb_nums;
2978e705c121SKalle Valo 	     i = (i + 1) & RX_QUEUE_MASK, j++) {
2979e705c121SKalle Valo 		struct iwl_rx_mem_buffer *rxb = rxq->queue[i];
2980e705c121SKalle Valo 		struct iwl_fw_error_dump_rb *rb;
2981e705c121SKalle Valo 
2982e705c121SKalle Valo 		dma_unmap_page(trans->dev, rxb->page_dma, max_len,
2983e705c121SKalle Valo 			       DMA_FROM_DEVICE);
2984e705c121SKalle Valo 
2985e705c121SKalle Valo 		rb_len += sizeof(**data) + sizeof(*rb) + max_len;
2986e705c121SKalle Valo 
2987e705c121SKalle Valo 		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RB);
2988e705c121SKalle Valo 		(*data)->len = cpu_to_le32(sizeof(*rb) + max_len);
2989e705c121SKalle Valo 		rb = (void *)(*data)->data;
2990e705c121SKalle Valo 		rb->index = cpu_to_le32(i);
2991e705c121SKalle Valo 		memcpy(rb->data, page_address(rxb->page), max_len);
2992e705c121SKalle Valo 		/* remap the page for the free benefit */
2993cfdc20efSJohannes Berg 		rxb->page_dma = dma_map_page(trans->dev, rxb->page,
2994cfdc20efSJohannes Berg 					     rxb->offset, max_len,
2995e705c121SKalle Valo 					     DMA_FROM_DEVICE);
2996e705c121SKalle Valo 
2997e705c121SKalle Valo 		*data = iwl_fw_error_next_data(*data);
2998e705c121SKalle Valo 	}
2999e705c121SKalle Valo 
3000e705c121SKalle Valo 	spin_unlock(&rxq->lock);
3001e705c121SKalle Valo 
3002e705c121SKalle Valo 	return rb_len;
3003e705c121SKalle Valo }
3004e705c121SKalle Valo #define IWL_CSR_TO_DUMP (0x250)
3005e705c121SKalle Valo 
3006e705c121SKalle Valo static u32 iwl_trans_pcie_dump_csr(struct iwl_trans *trans,
3007e705c121SKalle Valo 				   struct iwl_fw_error_dump_data **data)
3008e705c121SKalle Valo {
3009e705c121SKalle Valo 	u32 csr_len = sizeof(**data) + IWL_CSR_TO_DUMP;
3010e705c121SKalle Valo 	__le32 *val;
3011e705c121SKalle Valo 	int i;
3012e705c121SKalle Valo 
3013e705c121SKalle Valo 	(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_CSR);
3014e705c121SKalle Valo 	(*data)->len = cpu_to_le32(IWL_CSR_TO_DUMP);
3015e705c121SKalle Valo 	val = (void *)(*data)->data;
3016e705c121SKalle Valo 
3017e705c121SKalle Valo 	for (i = 0; i < IWL_CSR_TO_DUMP; i += 4)
3018e705c121SKalle Valo 		*val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
3019e705c121SKalle Valo 
3020e705c121SKalle Valo 	*data = iwl_fw_error_next_data(*data);
3021e705c121SKalle Valo 
3022e705c121SKalle Valo 	return csr_len;
3023e705c121SKalle Valo }
3024e705c121SKalle Valo 
3025e705c121SKalle Valo static u32 iwl_trans_pcie_fh_regs_dump(struct iwl_trans *trans,
3026e705c121SKalle Valo 				       struct iwl_fw_error_dump_data **data)
3027e705c121SKalle Valo {
3028e705c121SKalle Valo 	u32 fh_regs_len = FH_MEM_UPPER_BOUND - FH_MEM_LOWER_BOUND;
3029e705c121SKalle Valo 	unsigned long flags;
3030e705c121SKalle Valo 	__le32 *val;
3031e705c121SKalle Valo 	int i;
3032e705c121SKalle Valo 
303323ba9340SEmmanuel Grumbach 	if (!iwl_trans_grab_nic_access(trans, &flags))
3034e705c121SKalle Valo 		return 0;
3035e705c121SKalle Valo 
3036e705c121SKalle Valo 	(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FH_REGS);
3037e705c121SKalle Valo 	(*data)->len = cpu_to_le32(fh_regs_len);
3038e705c121SKalle Valo 	val = (void *)(*data)->data;
3039e705c121SKalle Valo 
3040286ca8ebSLuca Coelho 	if (!trans->trans_cfg->gen2)
3041723b45e2SLiad Kaufman 		for (i = FH_MEM_LOWER_BOUND; i < FH_MEM_UPPER_BOUND;
3042723b45e2SLiad Kaufman 		     i += sizeof(u32))
3043e705c121SKalle Valo 			*val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
3044723b45e2SLiad Kaufman 	else
3045ea695b7cSShaul Triebitz 		for (i = iwl_umac_prph(trans, FH_MEM_LOWER_BOUND_GEN2);
3046ea695b7cSShaul Triebitz 		     i < iwl_umac_prph(trans, FH_MEM_UPPER_BOUND_GEN2);
3047723b45e2SLiad Kaufman 		     i += sizeof(u32))
3048723b45e2SLiad Kaufman 			*val++ = cpu_to_le32(iwl_trans_pcie_read_prph(trans,
3049723b45e2SLiad Kaufman 								      i));
3050e705c121SKalle Valo 
3051e705c121SKalle Valo 	iwl_trans_release_nic_access(trans, &flags);
3052e705c121SKalle Valo 
3053e705c121SKalle Valo 	*data = iwl_fw_error_next_data(*data);
3054e705c121SKalle Valo 
3055e705c121SKalle Valo 	return sizeof(**data) + fh_regs_len;
3056e705c121SKalle Valo }
3057e705c121SKalle Valo 
3058e705c121SKalle Valo static u32
3059e705c121SKalle Valo iwl_trans_pci_dump_marbh_monitor(struct iwl_trans *trans,
3060e705c121SKalle Valo 				 struct iwl_fw_error_dump_fw_mon *fw_mon_data,
3061e705c121SKalle Valo 				 u32 monitor_len)
3062e705c121SKalle Valo {
3063e705c121SKalle Valo 	u32 buf_size_in_dwords = (monitor_len >> 2);
3064e705c121SKalle Valo 	u32 *buffer = (u32 *)fw_mon_data->data;
3065e705c121SKalle Valo 	unsigned long flags;
3066e705c121SKalle Valo 	u32 i;
3067e705c121SKalle Valo 
306823ba9340SEmmanuel Grumbach 	if (!iwl_trans_grab_nic_access(trans, &flags))
3069e705c121SKalle Valo 		return 0;
3070e705c121SKalle Valo 
3071ea695b7cSShaul Triebitz 	iwl_write_umac_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x1);
3072e705c121SKalle Valo 	for (i = 0; i < buf_size_in_dwords; i++)
3073ea695b7cSShaul Triebitz 		buffer[i] = iwl_read_umac_prph_no_grab(trans,
307414ef1b43SGolan Ben-Ami 						       MON_DMARB_RD_DATA_ADDR);
3075ea695b7cSShaul Triebitz 	iwl_write_umac_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x0);
3076e705c121SKalle Valo 
3077e705c121SKalle Valo 	iwl_trans_release_nic_access(trans, &flags);
3078e705c121SKalle Valo 
3079e705c121SKalle Valo 	return monitor_len;
3080e705c121SKalle Valo }
3081e705c121SKalle Valo 
30827a14c23dSSara Sharon static void
30837a14c23dSSara Sharon iwl_trans_pcie_dump_pointers(struct iwl_trans *trans,
30847a14c23dSSara Sharon 			     struct iwl_fw_error_dump_fw_mon *fw_mon_data)
30857a14c23dSSara Sharon {
3086c88580e1SShahar S Matityahu 	u32 base, base_high, write_ptr, write_ptr_val, wrap_cnt;
30877a14c23dSSara Sharon 
3088286ca8ebSLuca Coelho 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
3089c88580e1SShahar S Matityahu 		base = DBGC_CUR_DBGBUF_BASE_ADDR_LSB;
3090c88580e1SShahar S Matityahu 		base_high = DBGC_CUR_DBGBUF_BASE_ADDR_MSB;
3091c88580e1SShahar S Matityahu 		write_ptr = DBGC_CUR_DBGBUF_STATUS;
3092c88580e1SShahar S Matityahu 		wrap_cnt = DBGC_DBGBUF_WRAP_AROUND;
309391c28b83SShahar S Matityahu 	} else if (trans->dbg.dest_tlv) {
309491c28b83SShahar S Matityahu 		write_ptr = le32_to_cpu(trans->dbg.dest_tlv->write_ptr_reg);
309591c28b83SShahar S Matityahu 		wrap_cnt = le32_to_cpu(trans->dbg.dest_tlv->wrap_count);
309691c28b83SShahar S Matityahu 		base = le32_to_cpu(trans->dbg.dest_tlv->base_reg);
30977a14c23dSSara Sharon 	} else {
30987a14c23dSSara Sharon 		base = MON_BUFF_BASE_ADDR;
30997a14c23dSSara Sharon 		write_ptr = MON_BUFF_WRPTR;
31007a14c23dSSara Sharon 		wrap_cnt = MON_BUFF_CYCLE_CNT;
31017a14c23dSSara Sharon 	}
3102c88580e1SShahar S Matityahu 
3103c88580e1SShahar S Matityahu 	write_ptr_val = iwl_read_prph(trans, write_ptr);
31047a14c23dSSara Sharon 	fw_mon_data->fw_mon_cycle_cnt =
31057a14c23dSSara Sharon 		cpu_to_le32(iwl_read_prph(trans, wrap_cnt));
31067a14c23dSSara Sharon 	fw_mon_data->fw_mon_base_ptr =
31077a14c23dSSara Sharon 		cpu_to_le32(iwl_read_prph(trans, base));
3108286ca8ebSLuca Coelho 	if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
3109c88580e1SShahar S Matityahu 		fw_mon_data->fw_mon_base_high_ptr =
3110c88580e1SShahar S Matityahu 			cpu_to_le32(iwl_read_prph(trans, base_high));
3111c88580e1SShahar S Matityahu 		write_ptr_val &= DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK;
3112c88580e1SShahar S Matityahu 	}
3113c88580e1SShahar S Matityahu 	fw_mon_data->fw_mon_wr_ptr = cpu_to_le32(write_ptr_val);
31147a14c23dSSara Sharon }
31157a14c23dSSara Sharon 
3116e705c121SKalle Valo static u32
3117e705c121SKalle Valo iwl_trans_pcie_dump_monitor(struct iwl_trans *trans,
3118e705c121SKalle Valo 			    struct iwl_fw_error_dump_data **data,
3119e705c121SKalle Valo 			    u32 monitor_len)
3120e705c121SKalle Valo {
312169f0e505SShahar S Matityahu 	struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
3122e705c121SKalle Valo 	u32 len = 0;
3123e705c121SKalle Valo 
312491c28b83SShahar S Matityahu 	if (trans->dbg.dest_tlv ||
312569f0e505SShahar S Matityahu 	    (fw_mon->size &&
3126286ca8ebSLuca Coelho 	     (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000 ||
3127286ca8ebSLuca Coelho 	      trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210))) {
3128e705c121SKalle Valo 		struct iwl_fw_error_dump_fw_mon *fw_mon_data;
3129e705c121SKalle Valo 
3130e705c121SKalle Valo 		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FW_MONITOR);
3131e705c121SKalle Valo 		fw_mon_data = (void *)(*data)->data;
31327a14c23dSSara Sharon 
31337a14c23dSSara Sharon 		iwl_trans_pcie_dump_pointers(trans, fw_mon_data);
3134e705c121SKalle Valo 
3135e705c121SKalle Valo 		len += sizeof(**data) + sizeof(*fw_mon_data);
313669f0e505SShahar S Matityahu 		if (fw_mon->size) {
313769f0e505SShahar S Matityahu 			memcpy(fw_mon_data->data, fw_mon->block, fw_mon->size);
313869f0e505SShahar S Matityahu 			monitor_len = fw_mon->size;
313991c28b83SShahar S Matityahu 		} else if (trans->dbg.dest_tlv->monitor_mode == SMEM_MODE) {
31407a14c23dSSara Sharon 			u32 base = le32_to_cpu(fw_mon_data->fw_mon_base_ptr);
3141e705c121SKalle Valo 			/*
3142e705c121SKalle Valo 			 * Update pointers to reflect actual values after
3143e705c121SKalle Valo 			 * shifting
3144e705c121SKalle Valo 			 */
314591c28b83SShahar S Matityahu 			if (trans->dbg.dest_tlv->version) {
3146fd527eb5SGolan Ben Ami 				base = (iwl_read_prph(trans, base) &
3147fd527eb5SGolan Ben Ami 					IWL_LDBG_M2S_BUF_BA_MSK) <<
314891c28b83SShahar S Matityahu 				       trans->dbg.dest_tlv->base_shift;
3149fd527eb5SGolan Ben Ami 				base *= IWL_M2S_UNIT_SIZE;
3150fd527eb5SGolan Ben Ami 				base += trans->cfg->smem_offset;
3151fd527eb5SGolan Ben Ami 			} else {
3152e705c121SKalle Valo 				base = iwl_read_prph(trans, base) <<
315391c28b83SShahar S Matityahu 				       trans->dbg.dest_tlv->base_shift;
3154fd527eb5SGolan Ben Ami 			}
3155fd527eb5SGolan Ben Ami 
3156e705c121SKalle Valo 			iwl_trans_read_mem(trans, base, fw_mon_data->data,
3157e705c121SKalle Valo 					   monitor_len / sizeof(u32));
315891c28b83SShahar S Matityahu 		} else if (trans->dbg.dest_tlv->monitor_mode == MARBH_MODE) {
3159e705c121SKalle Valo 			monitor_len =
3160e705c121SKalle Valo 				iwl_trans_pci_dump_marbh_monitor(trans,
3161e705c121SKalle Valo 								 fw_mon_data,
3162e705c121SKalle Valo 								 monitor_len);
3163e705c121SKalle Valo 		} else {
3164e705c121SKalle Valo 			/* Didn't match anything - output no monitor data */
3165e705c121SKalle Valo 			monitor_len = 0;
3166e705c121SKalle Valo 		}
3167e705c121SKalle Valo 
3168e705c121SKalle Valo 		len += monitor_len;
3169e705c121SKalle Valo 		(*data)->len = cpu_to_le32(monitor_len + sizeof(*fw_mon_data));
3170e705c121SKalle Valo 	}
3171e705c121SKalle Valo 
3172e705c121SKalle Valo 	return len;
3173e705c121SKalle Valo }
3174e705c121SKalle Valo 
317593079fd5SJohannes Berg static int iwl_trans_get_fw_monitor_len(struct iwl_trans *trans, u32 *len)
3176e705c121SKalle Valo {
317769f0e505SShahar S Matityahu 	if (trans->dbg.fw_mon.size) {
3178da752717SShahar S Matityahu 		*len += sizeof(struct iwl_fw_error_dump_data) +
3179da752717SShahar S Matityahu 			sizeof(struct iwl_fw_error_dump_fw_mon) +
318069f0e505SShahar S Matityahu 			trans->dbg.fw_mon.size;
318169f0e505SShahar S Matityahu 		return trans->dbg.fw_mon.size;
318291c28b83SShahar S Matityahu 	} else if (trans->dbg.dest_tlv) {
3183da752717SShahar S Matityahu 		u32 base, end, cfg_reg, monitor_len;
3184e705c121SKalle Valo 
318591c28b83SShahar S Matityahu 		if (trans->dbg.dest_tlv->version == 1) {
318691c28b83SShahar S Matityahu 			cfg_reg = le32_to_cpu(trans->dbg.dest_tlv->base_reg);
3187fd527eb5SGolan Ben Ami 			cfg_reg = iwl_read_prph(trans, cfg_reg);
3188fd527eb5SGolan Ben Ami 			base = (cfg_reg & IWL_LDBG_M2S_BUF_BA_MSK) <<
318991c28b83SShahar S Matityahu 				trans->dbg.dest_tlv->base_shift;
3190fd527eb5SGolan Ben Ami 			base *= IWL_M2S_UNIT_SIZE;
3191fd527eb5SGolan Ben Ami 			base += trans->cfg->smem_offset;
3192fd527eb5SGolan Ben Ami 
3193fd527eb5SGolan Ben Ami 			monitor_len =
3194fd527eb5SGolan Ben Ami 				(cfg_reg & IWL_LDBG_M2S_BUF_SIZE_MSK) >>
319591c28b83SShahar S Matityahu 				trans->dbg.dest_tlv->end_shift;
3196fd527eb5SGolan Ben Ami 			monitor_len *= IWL_M2S_UNIT_SIZE;
3197fd527eb5SGolan Ben Ami 		} else {
319891c28b83SShahar S Matityahu 			base = le32_to_cpu(trans->dbg.dest_tlv->base_reg);
319991c28b83SShahar S Matityahu 			end = le32_to_cpu(trans->dbg.dest_tlv->end_reg);
3200e705c121SKalle Valo 
3201e705c121SKalle Valo 			base = iwl_read_prph(trans, base) <<
320291c28b83SShahar S Matityahu 			       trans->dbg.dest_tlv->base_shift;
3203e705c121SKalle Valo 			end = iwl_read_prph(trans, end) <<
320491c28b83SShahar S Matityahu 			      trans->dbg.dest_tlv->end_shift;
3205e705c121SKalle Valo 
3206e705c121SKalle Valo 			/* Make "end" point to the actual end */
3207286ca8ebSLuca Coelho 			if (trans->trans_cfg->device_family >=
3208fd527eb5SGolan Ben Ami 			    IWL_DEVICE_FAMILY_8000 ||
320991c28b83SShahar S Matityahu 			    trans->dbg.dest_tlv->monitor_mode == MARBH_MODE)
321091c28b83SShahar S Matityahu 				end += (1 << trans->dbg.dest_tlv->end_shift);
3211e705c121SKalle Valo 			monitor_len = end - base;
3212fd527eb5SGolan Ben Ami 		}
3213da752717SShahar S Matityahu 		*len += sizeof(struct iwl_fw_error_dump_data) +
3214da752717SShahar S Matityahu 			sizeof(struct iwl_fw_error_dump_fw_mon) +
3215e705c121SKalle Valo 			monitor_len;
3216da752717SShahar S Matityahu 		return monitor_len;
3217e705c121SKalle Valo 	}
3218da752717SShahar S Matityahu 	return 0;
3219da752717SShahar S Matityahu }
3220da752717SShahar S Matityahu 
3221da752717SShahar S Matityahu static struct iwl_trans_dump_data
3222da752717SShahar S Matityahu *iwl_trans_pcie_dump_data(struct iwl_trans *trans,
322379f033f6SSara Sharon 			  u32 dump_mask)
3224da752717SShahar S Matityahu {
3225da752717SShahar S Matityahu 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3226da752717SShahar S Matityahu 	struct iwl_fw_error_dump_data *data;
32274f4822b7SMordechay Goodstein 	struct iwl_txq *cmdq = trans->txqs.txq[trans->txqs.cmd.q_id];
3228da752717SShahar S Matityahu 	struct iwl_fw_error_dump_txcmd *txcmd;
3229da752717SShahar S Matityahu 	struct iwl_trans_dump_data *dump_data;
3230fefbf853SShahar S Matityahu 	u32 len, num_rbs = 0, monitor_len = 0;
3231da752717SShahar S Matityahu 	int i, ptr;
3232da752717SShahar S Matityahu 	bool dump_rbs = test_bit(STATUS_FW_ERROR, &trans->status) &&
3233286ca8ebSLuca Coelho 			!trans->trans_cfg->mq_rx_supported &&
323479f033f6SSara Sharon 			dump_mask & BIT(IWL_FW_ERROR_DUMP_RB);
323579f033f6SSara Sharon 
323679f033f6SSara Sharon 	if (!dump_mask)
323779f033f6SSara Sharon 		return NULL;
3238da752717SShahar S Matityahu 
3239da752717SShahar S Matityahu 	/* transport dump header */
3240da752717SShahar S Matityahu 	len = sizeof(*dump_data);
3241da752717SShahar S Matityahu 
3242da752717SShahar S Matityahu 	/* host commands */
3243e4eee943SShahar S Matityahu 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_TXCMD) && cmdq)
3244da752717SShahar S Matityahu 		len += sizeof(*data) +
32458672aad3SShahar S Matityahu 			cmdq->n_window * (sizeof(*txcmd) +
32468672aad3SShahar S Matityahu 					  TFD_MAX_PAYLOAD_SIZE);
3247da752717SShahar S Matityahu 
3248da752717SShahar S Matityahu 	/* FW monitor */
3249fefbf853SShahar S Matityahu 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FW_MONITOR))
3250da752717SShahar S Matityahu 		monitor_len = iwl_trans_get_fw_monitor_len(trans, &len);
3251e705c121SKalle Valo 
3252e705c121SKalle Valo 	/* CSR registers */
325379f033f6SSara Sharon 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR))
3254e705c121SKalle Valo 		len += sizeof(*data) + IWL_CSR_TO_DUMP;
3255e705c121SKalle Valo 
3256e705c121SKalle Valo 	/* FH registers */
325779f033f6SSara Sharon 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS)) {
3258286ca8ebSLuca Coelho 		if (trans->trans_cfg->gen2)
3259723b45e2SLiad Kaufman 			len += sizeof(*data) +
3260ea695b7cSShaul Triebitz 			       (iwl_umac_prph(trans, FH_MEM_UPPER_BOUND_GEN2) -
3261ea695b7cSShaul Triebitz 				iwl_umac_prph(trans, FH_MEM_LOWER_BOUND_GEN2));
3262723b45e2SLiad Kaufman 		else
3263723b45e2SLiad Kaufman 			len += sizeof(*data) +
3264520f03eaSShahar S Matityahu 			       (FH_MEM_UPPER_BOUND -
3265520f03eaSShahar S Matityahu 				FH_MEM_LOWER_BOUND);
3266520f03eaSShahar S Matityahu 	}
3267e705c121SKalle Valo 
3268e705c121SKalle Valo 	if (dump_rbs) {
326978485054SSara Sharon 		/* Dump RBs is supported only for pre-9000 devices (1 queue) */
327078485054SSara Sharon 		struct iwl_rxq *rxq = &trans_pcie->rxq[0];
3271e705c121SKalle Valo 		/* RBs */
32720307c839SGolan Ben Ami 		num_rbs =
32730307c839SGolan Ben Ami 			le16_to_cpu(iwl_get_closed_rb_stts(trans, rxq))
3274e705c121SKalle Valo 			& 0x0FFF;
327578485054SSara Sharon 		num_rbs = (num_rbs - rxq->read) & RX_QUEUE_MASK;
3276e705c121SKalle Valo 		len += num_rbs * (sizeof(*data) +
3277e705c121SKalle Valo 				  sizeof(struct iwl_fw_error_dump_rb) +
3278e705c121SKalle Valo 				  (PAGE_SIZE << trans_pcie->rx_page_order));
3279e705c121SKalle Valo 	}
3280e705c121SKalle Valo 
32815538409bSLiad Kaufman 	/* Paged memory for gen2 HW */
3282286ca8ebSLuca Coelho 	if (trans->trans_cfg->gen2 && dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING))
3283505a00c0SShahar S Matityahu 		for (i = 0; i < trans->init_dram.paging_cnt; i++)
32845538409bSLiad Kaufman 			len += sizeof(*data) +
32855538409bSLiad Kaufman 			       sizeof(struct iwl_fw_error_dump_paging) +
3286505a00c0SShahar S Matityahu 			       trans->init_dram.paging[i].size;
32875538409bSLiad Kaufman 
3288e705c121SKalle Valo 	dump_data = vzalloc(len);
3289e705c121SKalle Valo 	if (!dump_data)
3290e705c121SKalle Valo 		return NULL;
3291e705c121SKalle Valo 
3292e705c121SKalle Valo 	len = 0;
3293e705c121SKalle Valo 	data = (void *)dump_data->data;
3294520f03eaSShahar S Matityahu 
3295e4eee943SShahar S Matityahu 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_TXCMD) && cmdq) {
3296885375d0SMordechay Goodstein 		u16 tfd_size = trans->txqs.tfd.size;
3297520f03eaSShahar S Matityahu 
3298e705c121SKalle Valo 		data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXCMD);
3299e705c121SKalle Valo 		txcmd = (void *)data->data;
3300e705c121SKalle Valo 		spin_lock_bh(&cmdq->lock);
3301bb98ecd4SSara Sharon 		ptr = cmdq->write_ptr;
3302bb98ecd4SSara Sharon 		for (i = 0; i < cmdq->n_window; i++) {
33030cd1ad2dSMordechay Goodstein 			u8 idx = iwl_txq_get_cmd_index(cmdq, ptr);
330408326a97SJohannes Berg 			u8 tfdidx;
3305e705c121SKalle Valo 			u32 caplen, cmdlen;
3306e705c121SKalle Valo 
330708326a97SJohannes Berg 			if (trans->trans_cfg->use_tfh)
330808326a97SJohannes Berg 				tfdidx = idx;
330908326a97SJohannes Berg 			else
331008326a97SJohannes Berg 				tfdidx = ptr;
331108326a97SJohannes Berg 
3312520f03eaSShahar S Matityahu 			cmdlen = iwl_trans_pcie_get_cmdlen(trans,
331308326a97SJohannes Berg 							   (u8 *)cmdq->tfds +
331408326a97SJohannes Berg 							   tfd_size * tfdidx);
3315e705c121SKalle Valo 			caplen = min_t(u32, TFD_MAX_PAYLOAD_SIZE, cmdlen);
3316e705c121SKalle Valo 
3317e705c121SKalle Valo 			if (cmdlen) {
3318e705c121SKalle Valo 				len += sizeof(*txcmd) + caplen;
3319e705c121SKalle Valo 				txcmd->cmdlen = cpu_to_le32(cmdlen);
3320e705c121SKalle Valo 				txcmd->caplen = cpu_to_le32(caplen);
3321520f03eaSShahar S Matityahu 				memcpy(txcmd->data, cmdq->entries[idx].cmd,
3322520f03eaSShahar S Matityahu 				       caplen);
3323e705c121SKalle Valo 				txcmd = (void *)((u8 *)txcmd->data + caplen);
3324e705c121SKalle Valo 			}
3325e705c121SKalle Valo 
33260cd1ad2dSMordechay Goodstein 			ptr = iwl_txq_dec_wrap(trans, ptr);
3327e705c121SKalle Valo 		}
3328e705c121SKalle Valo 		spin_unlock_bh(&cmdq->lock);
3329e705c121SKalle Valo 
3330e705c121SKalle Valo 		data->len = cpu_to_le32(len);
3331e705c121SKalle Valo 		len += sizeof(*data);
3332e705c121SKalle Valo 		data = iwl_fw_error_next_data(data);
3333520f03eaSShahar S Matityahu 	}
3334e705c121SKalle Valo 
333579f033f6SSara Sharon 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR))
3336e705c121SKalle Valo 		len += iwl_trans_pcie_dump_csr(trans, &data);
333779f033f6SSara Sharon 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS))
3338e705c121SKalle Valo 		len += iwl_trans_pcie_fh_regs_dump(trans, &data);
3339e705c121SKalle Valo 	if (dump_rbs)
3340e705c121SKalle Valo 		len += iwl_trans_pcie_dump_rbs(trans, &data, num_rbs);
3341e705c121SKalle Valo 
33425538409bSLiad Kaufman 	/* Paged memory for gen2 HW */
3343286ca8ebSLuca Coelho 	if (trans->trans_cfg->gen2 &&
334479b6c8feSLuca Coelho 	    dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING)) {
3345505a00c0SShahar S Matityahu 		for (i = 0; i < trans->init_dram.paging_cnt; i++) {
33465538409bSLiad Kaufman 			struct iwl_fw_error_dump_paging *paging;
3347505a00c0SShahar S Matityahu 			u32 page_len = trans->init_dram.paging[i].size;
33485538409bSLiad Kaufman 
33495538409bSLiad Kaufman 			data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
33505538409bSLiad Kaufman 			data->len = cpu_to_le32(sizeof(*paging) + page_len);
33515538409bSLiad Kaufman 			paging = (void *)data->data;
33525538409bSLiad Kaufman 			paging->index = cpu_to_le32(i);
33535538409bSLiad Kaufman 			memcpy(paging->data,
3354505a00c0SShahar S Matityahu 			       trans->init_dram.paging[i].block, page_len);
33555538409bSLiad Kaufman 			data = iwl_fw_error_next_data(data);
33565538409bSLiad Kaufman 
33575538409bSLiad Kaufman 			len += sizeof(*data) + sizeof(*paging) + page_len;
33585538409bSLiad Kaufman 		}
33595538409bSLiad Kaufman 	}
336079f033f6SSara Sharon 	if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FW_MONITOR))
3361e705c121SKalle Valo 		len += iwl_trans_pcie_dump_monitor(trans, &data, monitor_len);
3362e705c121SKalle Valo 
3363e705c121SKalle Valo 	dump_data->len = len;
3364e705c121SKalle Valo 
3365e705c121SKalle Valo 	return dump_data;
3366e705c121SKalle Valo }
3367e705c121SKalle Valo 
33684cbb8e50SLuciano Coelho #ifdef CONFIG_PM_SLEEP
33694cbb8e50SLuciano Coelho static int iwl_trans_pcie_suspend(struct iwl_trans *trans)
33704cbb8e50SLuciano Coelho {
33714cbb8e50SLuciano Coelho 	return 0;
33724cbb8e50SLuciano Coelho }
33734cbb8e50SLuciano Coelho 
33744cbb8e50SLuciano Coelho static void iwl_trans_pcie_resume(struct iwl_trans *trans)
33754cbb8e50SLuciano Coelho {
33764cbb8e50SLuciano Coelho }
33774cbb8e50SLuciano Coelho #endif /* CONFIG_PM_SLEEP */
33784cbb8e50SLuciano Coelho 
3379623e7766SSara Sharon #define IWL_TRANS_COMMON_OPS						\
3380623e7766SSara Sharon 	.op_mode_leave = iwl_trans_pcie_op_mode_leave,			\
3381623e7766SSara Sharon 	.write8 = iwl_trans_pcie_write8,				\
3382623e7766SSara Sharon 	.write32 = iwl_trans_pcie_write32,				\
3383623e7766SSara Sharon 	.read32 = iwl_trans_pcie_read32,				\
3384623e7766SSara Sharon 	.read_prph = iwl_trans_pcie_read_prph,				\
3385623e7766SSara Sharon 	.write_prph = iwl_trans_pcie_write_prph,			\
3386623e7766SSara Sharon 	.read_mem = iwl_trans_pcie_read_mem,				\
3387623e7766SSara Sharon 	.write_mem = iwl_trans_pcie_write_mem,				\
33887f1fe1d4SLuca Coelho 	.read_config32 = iwl_trans_pcie_read_config32,			\
3389623e7766SSara Sharon 	.configure = iwl_trans_pcie_configure,				\
3390623e7766SSara Sharon 	.set_pmi = iwl_trans_pcie_set_pmi,				\
3391870c2a11SGolan Ben Ami 	.sw_reset = iwl_trans_pcie_sw_reset,				\
3392623e7766SSara Sharon 	.grab_nic_access = iwl_trans_pcie_grab_nic_access,		\
3393623e7766SSara Sharon 	.release_nic_access = iwl_trans_pcie_release_nic_access,	\
3394623e7766SSara Sharon 	.set_bits_mask = iwl_trans_pcie_set_bits_mask,			\
3395623e7766SSara Sharon 	.dump_data = iwl_trans_pcie_dump_data,				\
3396623e7766SSara Sharon 	.d3_suspend = iwl_trans_pcie_d3_suspend,			\
3397d1967ce6SShahar S Matityahu 	.d3_resume = iwl_trans_pcie_d3_resume,				\
3398d1967ce6SShahar S Matityahu 	.sync_nmi = iwl_trans_pcie_sync_nmi
3399623e7766SSara Sharon 
3400623e7766SSara Sharon #ifdef CONFIG_PM_SLEEP
3401623e7766SSara Sharon #define IWL_TRANS_PM_OPS						\
3402623e7766SSara Sharon 	.suspend = iwl_trans_pcie_suspend,				\
3403623e7766SSara Sharon 	.resume = iwl_trans_pcie_resume,
3404623e7766SSara Sharon #else
3405623e7766SSara Sharon #define IWL_TRANS_PM_OPS
3406623e7766SSara Sharon #endif /* CONFIG_PM_SLEEP */
3407623e7766SSara Sharon 
3408e705c121SKalle Valo static const struct iwl_trans_ops trans_ops_pcie = {
3409623e7766SSara Sharon 	IWL_TRANS_COMMON_OPS,
3410623e7766SSara Sharon 	IWL_TRANS_PM_OPS
3411e705c121SKalle Valo 	.start_hw = iwl_trans_pcie_start_hw,
3412e705c121SKalle Valo 	.fw_alive = iwl_trans_pcie_fw_alive,
3413e705c121SKalle Valo 	.start_fw = iwl_trans_pcie_start_fw,
3414e705c121SKalle Valo 	.stop_device = iwl_trans_pcie_stop_device,
3415e705c121SKalle Valo 
3416e705c121SKalle Valo 	.send_cmd = iwl_trans_pcie_send_hcmd,
3417e705c121SKalle Valo 
3418e705c121SKalle Valo 	.tx = iwl_trans_pcie_tx,
3419e705c121SKalle Valo 	.reclaim = iwl_trans_pcie_reclaim,
3420e705c121SKalle Valo 
3421e705c121SKalle Valo 	.txq_disable = iwl_trans_pcie_txq_disable,
3422e705c121SKalle Valo 	.txq_enable = iwl_trans_pcie_txq_enable,
3423e705c121SKalle Valo 
342442db09c1SLiad Kaufman 	.txq_set_shared_mode = iwl_trans_pcie_txq_set_shared_mode,
342542db09c1SLiad Kaufman 
3426d6d517b7SSara Sharon 	.wait_tx_queues_empty = iwl_trans_pcie_wait_txqs_empty,
3427d6d517b7SSara Sharon 
3428e705c121SKalle Valo 	.freeze_txq_timer = iwl_trans_pcie_freeze_txq_timer,
34290cd58eaaSEmmanuel Grumbach 	.block_txq_ptrs = iwl_trans_pcie_block_txq_ptrs,
3430f7805b33SLior Cohen #ifdef CONFIG_IWLWIFI_DEBUGFS
3431f7805b33SLior Cohen 	.debugfs_cleanup = iwl_trans_pcie_debugfs_cleanup,
3432f7805b33SLior Cohen #endif
3433623e7766SSara Sharon };
3434e705c121SKalle Valo 
3435623e7766SSara Sharon static const struct iwl_trans_ops trans_ops_pcie_gen2 = {
3436623e7766SSara Sharon 	IWL_TRANS_COMMON_OPS,
3437623e7766SSara Sharon 	IWL_TRANS_PM_OPS
3438623e7766SSara Sharon 	.start_hw = iwl_trans_pcie_start_hw,
3439eda50cdeSSara Sharon 	.fw_alive = iwl_trans_pcie_gen2_fw_alive,
3440eda50cdeSSara Sharon 	.start_fw = iwl_trans_pcie_gen2_start_fw,
344177c09bc8SSara Sharon 	.stop_device = iwl_trans_pcie_gen2_stop_device,
3442e705c121SKalle Valo 
3443ca60da2eSSara Sharon 	.send_cmd = iwl_trans_pcie_gen2_send_hcmd,
3444e705c121SKalle Valo 
34450cd1ad2dSMordechay Goodstein 	.tx = iwl_txq_gen2_tx,
3446623e7766SSara Sharon 	.reclaim = iwl_trans_pcie_reclaim,
3447623e7766SSara Sharon 
3448ba7136f3SAlex Malamud 	.set_q_ptrs = iwl_trans_pcie_set_q_ptrs,
3449ba7136f3SAlex Malamud 
34500cd1ad2dSMordechay Goodstein 	.txq_alloc = iwl_txq_dyn_alloc,
34510cd1ad2dSMordechay Goodstein 	.txq_free = iwl_txq_dyn_free,
3452d6d517b7SSara Sharon 	.wait_txq_empty = iwl_trans_pcie_wait_txq_empty,
345392536c96SSara Sharon 	.rxq_dma_data = iwl_trans_pcie_rxq_dma_data,
3454f7805b33SLior Cohen #ifdef CONFIG_IWLWIFI_DEBUGFS
3455f7805b33SLior Cohen 	.debugfs_cleanup = iwl_trans_pcie_debugfs_cleanup,
3456f7805b33SLior Cohen #endif
3457e705c121SKalle Valo };
3458e705c121SKalle Valo 
3459e705c121SKalle Valo struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
3460e705c121SKalle Valo 			       const struct pci_device_id *ent,
34617e8258c0SLuca Coelho 			       const struct iwl_cfg_trans_params *cfg_trans)
3462e705c121SKalle Valo {
3463e705c121SKalle Valo 	struct iwl_trans_pcie *trans_pcie;
3464e705c121SKalle Valo 	struct iwl_trans *trans;
3465fda1bd0dSMordechay Goodstein 	int ret, addr_size;
3466a89c72ffSJohannes Berg 	const struct iwl_trans_ops *ops = &trans_ops_pcie_gen2;
3467a89c72ffSJohannes Berg 
3468fda1bd0dSMordechay Goodstein 	if (!cfg_trans->gen2)
3469a89c72ffSJohannes Berg 		ops = &trans_ops_pcie;
3470e705c121SKalle Valo 
34715a41a86cSSharon Dvir 	ret = pcim_enable_device(pdev);
34725a41a86cSSharon Dvir 	if (ret)
34735a41a86cSSharon Dvir 		return ERR_PTR(ret);
34745a41a86cSSharon Dvir 
3475a89c72ffSJohannes Berg 	trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie), &pdev->dev, ops,
3476fda1bd0dSMordechay Goodstein 				cfg_trans);
3477e705c121SKalle Valo 	if (!trans)
3478e705c121SKalle Valo 		return ERR_PTR(-ENOMEM);
3479e705c121SKalle Valo 
3480e705c121SKalle Valo 	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3481e705c121SKalle Valo 
3482e705c121SKalle Valo 	trans_pcie->trans = trans;
3483326477e4SJohannes Berg 	trans_pcie->opmode_down = true;
3484e705c121SKalle Valo 	spin_lock_init(&trans_pcie->irq_lock);
3485e705c121SKalle Valo 	spin_lock_init(&trans_pcie->reg_lock);
3486cfdc20efSJohannes Berg 	spin_lock_init(&trans_pcie->alloc_page_lock);
3487e705c121SKalle Valo 	mutex_init(&trans_pcie->mutex);
3488e705c121SKalle Valo 	init_waitqueue_head(&trans_pcie->ucode_write_waitq);
34898188a18eSJohannes Berg 
34908188a18eSJohannes Berg 	trans_pcie->rba.alloc_wq = alloc_workqueue("rb_allocator",
34918188a18eSJohannes Berg 						   WQ_HIGHPRI | WQ_UNBOUND, 1);
34928188a18eSJohannes Berg 	if (!trans_pcie->rba.alloc_wq) {
34938188a18eSJohannes Berg 		ret = -ENOMEM;
34948188a18eSJohannes Berg 		goto out_free_trans;
34958188a18eSJohannes Berg 	}
34968188a18eSJohannes Berg 	INIT_WORK(&trans_pcie->rba.rx_alloc, iwl_pcie_rx_allocator_work);
34978188a18eSJohannes Berg 
3498c5bf4fa1SJohannes Berg 	trans_pcie->debug_rfkill = -1;
3499e705c121SKalle Valo 
35007e8258c0SLuca Coelho 	if (!cfg_trans->base_params->pcie_l1_allowed) {
3501e705c121SKalle Valo 		/*
3502e705c121SKalle Valo 		 * W/A - seems to solve weird behavior. We need to remove this
3503e705c121SKalle Valo 		 * if we don't want to stay in L1 all the time. This wastes a
3504e705c121SKalle Valo 		 * lot of power.
3505e705c121SKalle Valo 		 */
3506e705c121SKalle Valo 		pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S |
3507e705c121SKalle Valo 				       PCIE_LINK_STATE_L1 |
3508e705c121SKalle Valo 				       PCIE_LINK_STATE_CLKPM);
3509e705c121SKalle Valo 	}
3510e705c121SKalle Valo 
35119416560eSGolan Ben Ami 	trans_pcie->def_rx_queue = 0;
35129416560eSGolan Ben Ami 
3513e705c121SKalle Valo 	pci_set_master(pdev);
3514e705c121SKalle Valo 
3515885375d0SMordechay Goodstein 	addr_size = trans->txqs.tfd.addr_size;
351696a6497bSSara Sharon 	ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(addr_size));
3517e705c121SKalle Valo 	if (!ret)
351896a6497bSSara Sharon 		ret = pci_set_consistent_dma_mask(pdev,
351996a6497bSSara Sharon 						  DMA_BIT_MASK(addr_size));
3520e705c121SKalle Valo 	if (ret) {
3521e705c121SKalle Valo 		ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3522e705c121SKalle Valo 		if (!ret)
3523e705c121SKalle Valo 			ret = pci_set_consistent_dma_mask(pdev,
3524e705c121SKalle Valo 							  DMA_BIT_MASK(32));
3525e705c121SKalle Valo 		/* both attempts failed: */
3526e705c121SKalle Valo 		if (ret) {
3527e705c121SKalle Valo 			dev_err(&pdev->dev, "No suitable DMA available\n");
35285a41a86cSSharon Dvir 			goto out_no_pci;
3529e705c121SKalle Valo 		}
3530e705c121SKalle Valo 	}
3531e705c121SKalle Valo 
35325a41a86cSSharon Dvir 	ret = pcim_iomap_regions_request_all(pdev, BIT(0), DRV_NAME);
3533e705c121SKalle Valo 	if (ret) {
35345a41a86cSSharon Dvir 		dev_err(&pdev->dev, "pcim_iomap_regions_request_all failed\n");
35355a41a86cSSharon Dvir 		goto out_no_pci;
3536e705c121SKalle Valo 	}
3537e705c121SKalle Valo 
35385a41a86cSSharon Dvir 	trans_pcie->hw_base = pcim_iomap_table(pdev)[0];
3539e705c121SKalle Valo 	if (!trans_pcie->hw_base) {
35405a41a86cSSharon Dvir 		dev_err(&pdev->dev, "pcim_iomap_table failed\n");
3541e705c121SKalle Valo 		ret = -ENODEV;
35425a41a86cSSharon Dvir 		goto out_no_pci;
3543e705c121SKalle Valo 	}
3544e705c121SKalle Valo 
3545e705c121SKalle Valo 	/* We disable the RETRY_TIMEOUT register (0x41) to keep
3546e705c121SKalle Valo 	 * PCI Tx retries from interfering with C3 CPU state */
3547e705c121SKalle Valo 	pci_write_config_byte(pdev, PCI_CFG_RETRY_TIMEOUT, 0x00);
3548e705c121SKalle Valo 
3549e705c121SKalle Valo 	trans_pcie->pci_dev = pdev;
3550e705c121SKalle Valo 	iwl_disable_interrupts(trans);
3551e705c121SKalle Valo 
3552e705c121SKalle Valo 	trans->hw_rev = iwl_read32(trans, CSR_HW_REV);
35539a098a89SRajat Jain 	if (trans->hw_rev == 0xffffffff) {
35549a098a89SRajat Jain 		dev_err(&pdev->dev, "HW_REV=0xFFFFFFFF, PCI issues?\n");
35559a098a89SRajat Jain 		ret = -EIO;
35569a098a89SRajat Jain 		goto out_no_pci;
35579a098a89SRajat Jain 	}
35589a098a89SRajat Jain 
3559e705c121SKalle Valo 	/*
3560e705c121SKalle Valo 	 * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
3561e705c121SKalle Valo 	 * changed, and now the revision step also includes bit 0-1 (no more
3562e705c121SKalle Valo 	 * "dash" value). To keep hw_rev backwards compatible - we'll store it
3563e705c121SKalle Valo 	 * in the old format.
3564e705c121SKalle Valo 	 */
35657e8258c0SLuca Coelho 	if (cfg_trans->device_family >= IWL_DEVICE_FAMILY_8000) {
3566e705c121SKalle Valo 		trans->hw_rev = (trans->hw_rev & 0xfff0) |
3567e705c121SKalle Valo 				(CSR_HW_REV_STEP(trans->hw_rev << 2) << 2);
3568e705c121SKalle Valo 
3569e705c121SKalle Valo 		ret = iwl_pcie_prepare_card_hw(trans);
3570e705c121SKalle Valo 		if (ret) {
3571e705c121SKalle Valo 			IWL_WARN(trans, "Exit HW not ready\n");
35725a41a86cSSharon Dvir 			goto out_no_pci;
3573e705c121SKalle Valo 		}
3574e705c121SKalle Valo 
3575e705c121SKalle Valo 		/*
3576e705c121SKalle Valo 		 * in-order to recognize C step driver should read chip version
3577e705c121SKalle Valo 		 * id located at the AUX bus MISC address space.
3578e705c121SKalle Valo 		 */
35797e8258c0SLuca Coelho 		ret = iwl_finish_nic_init(trans, cfg_trans);
3580c96b5eecSJohannes Berg 		if (ret)
35815a41a86cSSharon Dvir 			goto out_no_pci;
3582e705c121SKalle Valo 
3583e705c121SKalle Valo 	}
3584e705c121SKalle Valo 
358599be6166SLuca Coelho 	IWL_DEBUG_INFO(trans, "HW REV: 0x%0x\n", trans->hw_rev);
358699be6166SLuca Coelho 
35877e8258c0SLuca Coelho 	iwl_pcie_set_interrupt_capa(pdev, trans, cfg_trans);
3588e705c121SKalle Valo 	trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
3589e705c121SKalle Valo 	snprintf(trans->hw_id_str, sizeof(trans->hw_id_str),
3590e705c121SKalle Valo 		 "PCI ID: 0x%04X:0x%04X", pdev->device, pdev->subsystem_device);
3591e705c121SKalle Valo 
3592e705c121SKalle Valo 	/* Initialize the wait queue for commands */
3593e705c121SKalle Valo 	init_waitqueue_head(&trans_pcie->wait_command_queue);
3594e705c121SKalle Valo 
3595e5f3f215SHaim Dreyfuss 	init_waitqueue_head(&trans_pcie->sx_waitq);
3596e5f3f215SHaim Dreyfuss 
3597c239feecSJohannes Berg 
35982e5d4a8fSHaim Dreyfuss 	if (trans_pcie->msix_enabled) {
35992388bd7bSDan Carpenter 		ret = iwl_pcie_init_msix_handler(pdev, trans_pcie);
36002388bd7bSDan Carpenter 		if (ret)
36015a41a86cSSharon Dvir 			goto out_no_pci;
36022e5d4a8fSHaim Dreyfuss 	 } else {
3603e705c121SKalle Valo 		ret = iwl_pcie_alloc_ict(trans);
3604e705c121SKalle Valo 		if (ret)
36055a41a86cSSharon Dvir 			goto out_no_pci;
3606e705c121SKalle Valo 
36075a41a86cSSharon Dvir 		ret = devm_request_threaded_irq(&pdev->dev, pdev->irq,
36085a41a86cSSharon Dvir 						iwl_pcie_isr,
3609e705c121SKalle Valo 						iwl_pcie_irq_handler,
3610e705c121SKalle Valo 						IRQF_SHARED, DRV_NAME, trans);
3611e705c121SKalle Valo 		if (ret) {
3612e705c121SKalle Valo 			IWL_ERR(trans, "Error allocating IRQ %d\n", pdev->irq);
3613e705c121SKalle Valo 			goto out_free_ict;
3614e705c121SKalle Valo 		}
3615e705c121SKalle Valo 		trans_pcie->inta_mask = CSR_INI_SET_MASK;
36162e5d4a8fSHaim Dreyfuss 	 }
3617e705c121SKalle Valo 
3618f7805b33SLior Cohen #ifdef CONFIG_IWLWIFI_DEBUGFS
3619f7805b33SLior Cohen 	trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED;
3620f7805b33SLior Cohen 	mutex_init(&trans_pcie->fw_mon_data.mutex);
3621f7805b33SLior Cohen #endif
3622f7805b33SLior Cohen 
3623a9248de4SShahar S Matityahu 	iwl_dbg_tlv_init(trans);
3624a9248de4SShahar S Matityahu 
3625e705c121SKalle Valo 	return trans;
3626e705c121SKalle Valo 
3627e705c121SKalle Valo out_free_ict:
3628e705c121SKalle Valo 	iwl_pcie_free_ict(trans);
3629e705c121SKalle Valo out_no_pci:
36308188a18eSJohannes Berg 	destroy_workqueue(trans_pcie->rba.alloc_wq);
36318188a18eSJohannes Berg out_free_trans:
3632e705c121SKalle Valo 	iwl_trans_free(trans);
3633e705c121SKalle Valo 	return ERR_PTR(ret);
3634e705c121SKalle Valo }
3635b8a7547dSShahar S Matityahu 
3636d1967ce6SShahar S Matityahu void iwl_trans_pcie_sync_nmi(struct iwl_trans *trans)
3637b8a7547dSShahar S Matityahu {
36381c6bca6dSShahar S Matityahu 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3639b8a7547dSShahar S Matityahu 	unsigned long timeout = jiffies + IWL_TRANS_NMI_TIMEOUT;
3640e4eee943SShahar S Matityahu 	bool interrupts_enabled = test_bit(STATUS_INT_ENABLED, &trans->status);
36411c6bca6dSShahar S Matityahu 	u32 inta_addr, sw_err_bit;
36421c6bca6dSShahar S Matityahu 
36431c6bca6dSShahar S Matityahu 	if (trans_pcie->msix_enabled) {
36441c6bca6dSShahar S Matityahu 		inta_addr = CSR_MSIX_HW_INT_CAUSES_AD;
36451c6bca6dSShahar S Matityahu 		sw_err_bit = MSIX_HW_INT_CAUSES_REG_SW_ERR;
36461c6bca6dSShahar S Matityahu 	} else {
36471c6bca6dSShahar S Matityahu 		inta_addr = CSR_INT;
36481c6bca6dSShahar S Matityahu 		sw_err_bit = CSR_INT_BIT_SW_ERR;
36491c6bca6dSShahar S Matityahu 	}
3650b8a7547dSShahar S Matityahu 
3651e4eee943SShahar S Matityahu 	/* if the interrupts were already disabled, there is no point in
3652e4eee943SShahar S Matityahu 	 * calling iwl_disable_interrupts
3653e4eee943SShahar S Matityahu 	 */
3654e4eee943SShahar S Matityahu 	if (interrupts_enabled)
3655b8a7547dSShahar S Matityahu 		iwl_disable_interrupts(trans);
3656e4eee943SShahar S Matityahu 
3657b8a7547dSShahar S Matityahu 	iwl_force_nmi(trans);
3658b8a7547dSShahar S Matityahu 	while (time_after(timeout, jiffies)) {
36591c6bca6dSShahar S Matityahu 		u32 inta_hw = iwl_read32(trans, inta_addr);
3660b8a7547dSShahar S Matityahu 
3661b8a7547dSShahar S Matityahu 		/* Error detected by uCode */
36621c6bca6dSShahar S Matityahu 		if (inta_hw & sw_err_bit) {
3663b8a7547dSShahar S Matityahu 			/* Clear causes register */
36641c6bca6dSShahar S Matityahu 			iwl_write32(trans, inta_addr, inta_hw & sw_err_bit);
3665b8a7547dSShahar S Matityahu 			break;
3666b8a7547dSShahar S Matityahu 		}
3667b8a7547dSShahar S Matityahu 
3668b8a7547dSShahar S Matityahu 		mdelay(1);
3669b8a7547dSShahar S Matityahu 	}
3670e4eee943SShahar S Matityahu 
3671e4eee943SShahar S Matityahu 	/* enable interrupts only if there were already enabled before this
3672e4eee943SShahar S Matityahu 	 * function to avoid a case were the driver enable interrupts before
3673e4eee943SShahar S Matityahu 	 * proper configurations were made
3674e4eee943SShahar S Matityahu 	 */
3675e4eee943SShahar S Matityahu 	if (interrupts_enabled)
3676b8a7547dSShahar S Matityahu 		iwl_enable_interrupts(trans);
3677e4eee943SShahar S Matityahu 
3678b8a7547dSShahar S Matityahu 	iwl_trans_fw_error(trans);
3679b8a7547dSShahar S Matityahu }
3680