1 // SPDX-License-Identifier: BSD-3-Clause-Clear
2 /*
3  * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
4  * Copyright (c) 2021-2022, Qualcomm Innovation Center, Inc. All rights reserved.
5  */
6 
7 #include "core.h"
8 #include "pcic.h"
9 #include "debug.h"
10 
11 static const char *irq_name[ATH11K_IRQ_NUM_MAX] = {
12 	"bhi",
13 	"mhi-er0",
14 	"mhi-er1",
15 	"ce0",
16 	"ce1",
17 	"ce2",
18 	"ce3",
19 	"ce4",
20 	"ce5",
21 	"ce6",
22 	"ce7",
23 	"ce8",
24 	"ce9",
25 	"ce10",
26 	"ce11",
27 	"host2wbm-desc-feed",
28 	"host2reo-re-injection",
29 	"host2reo-command",
30 	"host2rxdma-monitor-ring3",
31 	"host2rxdma-monitor-ring2",
32 	"host2rxdma-monitor-ring1",
33 	"reo2ost-exception",
34 	"wbm2host-rx-release",
35 	"reo2host-status",
36 	"reo2host-destination-ring4",
37 	"reo2host-destination-ring3",
38 	"reo2host-destination-ring2",
39 	"reo2host-destination-ring1",
40 	"rxdma2host-monitor-destination-mac3",
41 	"rxdma2host-monitor-destination-mac2",
42 	"rxdma2host-monitor-destination-mac1",
43 	"ppdu-end-interrupts-mac3",
44 	"ppdu-end-interrupts-mac2",
45 	"ppdu-end-interrupts-mac1",
46 	"rxdma2host-monitor-status-ring-mac3",
47 	"rxdma2host-monitor-status-ring-mac2",
48 	"rxdma2host-monitor-status-ring-mac1",
49 	"host2rxdma-host-buf-ring-mac3",
50 	"host2rxdma-host-buf-ring-mac2",
51 	"host2rxdma-host-buf-ring-mac1",
52 	"rxdma2host-destination-ring-mac3",
53 	"rxdma2host-destination-ring-mac2",
54 	"rxdma2host-destination-ring-mac1",
55 	"host2tcl-input-ring4",
56 	"host2tcl-input-ring3",
57 	"host2tcl-input-ring2",
58 	"host2tcl-input-ring1",
59 	"wbm2host-tx-completions-ring3",
60 	"wbm2host-tx-completions-ring2",
61 	"wbm2host-tx-completions-ring1",
62 	"tcl2host-status-ring",
63 };
64 
65 static const struct ath11k_msi_config ath11k_msi_config[] = {
66 	{
67 		.total_vectors = 32,
68 		.total_users = 4,
69 		.users = (struct ath11k_msi_user[]) {
70 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
71 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
72 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
73 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
74 		},
75 		.hw_rev = ATH11K_HW_QCA6390_HW20,
76 	},
77 	{
78 		.total_vectors = 16,
79 		.total_users = 3,
80 		.users = (struct ath11k_msi_user[]) {
81 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
82 			{ .name = "CE", .num_vectors = 5, .base_vector = 3 },
83 			{ .name = "DP", .num_vectors = 8, .base_vector = 8 },
84 		},
85 		.hw_rev = ATH11K_HW_QCN9074_HW10,
86 	},
87 	{
88 		.total_vectors = 32,
89 		.total_users = 4,
90 		.users = (struct ath11k_msi_user[]) {
91 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
92 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
93 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
94 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
95 		},
96 		.hw_rev = ATH11K_HW_WCN6855_HW20,
97 	},
98 	{
99 		.total_vectors = 32,
100 		.total_users = 4,
101 		.users = (struct ath11k_msi_user[]) {
102 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
103 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
104 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
105 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
106 		},
107 		.hw_rev = ATH11K_HW_WCN6855_HW21,
108 	},
109 	{
110 		.total_vectors = 28,
111 		.total_users = 2,
112 		.users = (struct ath11k_msi_user[]) {
113 			{ .name = "CE", .num_vectors = 10, .base_vector = 0 },
114 			{ .name = "DP", .num_vectors = 18, .base_vector = 10 },
115 		},
116 		.hw_rev = ATH11K_HW_WCN6750_HW10,
117 	},
118 };
119 
120 int ath11k_pcic_init_msi_config(struct ath11k_base *ab)
121 {
122 	const struct ath11k_msi_config *msi_config;
123 	int i;
124 
125 	for (i = 0; i < ARRAY_SIZE(ath11k_msi_config); i++) {
126 		msi_config = &ath11k_msi_config[i];
127 
128 		if (msi_config->hw_rev == ab->hw_rev)
129 			break;
130 	}
131 
132 	if (i == ARRAY_SIZE(ath11k_msi_config)) {
133 		ath11k_err(ab, "failed to fetch msi config, unsupported hw version: 0x%x\n",
134 			   ab->hw_rev);
135 		return -EINVAL;
136 	}
137 
138 	ab->pci.msi.config = msi_config;
139 	return 0;
140 }
141 EXPORT_SYMBOL(ath11k_pcic_init_msi_config);
142 
143 static void __ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value)
144 {
145 	if (offset < ATH11K_PCI_WINDOW_START)
146 		iowrite32(value, ab->mem  + offset);
147 	else
148 		ab->pci.ops->window_write32(ab, offset, value);
149 }
150 
151 void ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value)
152 {
153 	int ret = 0;
154 	bool wakeup_required;
155 
156 	/* for offset beyond BAR + 4K - 32, may
157 	 * need to wakeup the device to access.
158 	 */
159 	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
160 			  offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
161 	if (wakeup_required && ab->pci.ops->wakeup)
162 		ret = ab->pci.ops->wakeup(ab);
163 
164 	__ath11k_pcic_write32(ab, offset, value);
165 
166 	if (wakeup_required && !ret && ab->pci.ops->release)
167 		ab->pci.ops->release(ab);
168 }
169 EXPORT_SYMBOL(ath11k_pcic_write32);
170 
171 static u32 __ath11k_pcic_read32(struct ath11k_base *ab, u32 offset)
172 {
173 	u32 val;
174 
175 	if (offset < ATH11K_PCI_WINDOW_START)
176 		val = ioread32(ab->mem + offset);
177 	else
178 		val = ab->pci.ops->window_read32(ab, offset);
179 
180 	return val;
181 }
182 
183 u32 ath11k_pcic_read32(struct ath11k_base *ab, u32 offset)
184 {
185 	int ret = 0;
186 	u32 val;
187 	bool wakeup_required;
188 
189 	/* for offset beyond BAR + 4K - 32, may
190 	 * need to wakeup the device to access.
191 	 */
192 	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
193 			  offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
194 	if (wakeup_required && ab->pci.ops->wakeup)
195 		ret = ab->pci.ops->wakeup(ab);
196 
197 	val = __ath11k_pcic_read32(ab, offset);
198 
199 	if (wakeup_required && !ret && ab->pci.ops->release)
200 		ab->pci.ops->release(ab);
201 
202 	return val;
203 }
204 EXPORT_SYMBOL(ath11k_pcic_read32);
205 
206 int ath11k_pcic_read(struct ath11k_base *ab, void *buf, u32 start, u32 end)
207 {
208 	int ret = 0;
209 	bool wakeup_required;
210 	u32 *data = buf;
211 	u32 i;
212 
213 	/* for offset beyond BAR + 4K - 32, may
214 	 * need to wakeup the device to access.
215 	 */
216 	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
217 			  end >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
218 	if (wakeup_required && ab->pci.ops->wakeup) {
219 		ret = ab->pci.ops->wakeup(ab);
220 		if (ret) {
221 			ath11k_warn(ab,
222 				    "wakeup failed, data may be invalid: %d",
223 				    ret);
224 			/* Even though wakeup() failed, continue processing rather
225 			 * than returning because some parts of the data may still
226 			 * be valid and useful in some cases, e.g. could give us
227 			 * some clues on firmware crash.
228 			 * Mislead due to invalid data could be avoided because we
229 			 * are aware of the wakeup failure.
230 			 */
231 		}
232 	}
233 
234 	for (i = start; i < end + 1; i += 4)
235 		*data++ = __ath11k_pcic_read32(ab, i);
236 
237 	if (wakeup_required && ab->pci.ops->release)
238 		ab->pci.ops->release(ab);
239 
240 	return 0;
241 }
242 EXPORT_SYMBOL(ath11k_pcic_read);
243 
244 void ath11k_pcic_get_msi_address(struct ath11k_base *ab, u32 *msi_addr_lo,
245 				 u32 *msi_addr_hi)
246 {
247 	*msi_addr_lo = ab->pci.msi.addr_lo;
248 	*msi_addr_hi = ab->pci.msi.addr_hi;
249 }
250 EXPORT_SYMBOL(ath11k_pcic_get_msi_address);
251 
252 int ath11k_pcic_get_user_msi_assignment(struct ath11k_base *ab, char *user_name,
253 					int *num_vectors, u32 *user_base_data,
254 					u32 *base_vector)
255 {
256 	const struct ath11k_msi_config *msi_config = ab->pci.msi.config;
257 	int idx;
258 
259 	for (idx = 0; idx < msi_config->total_users; idx++) {
260 		if (strcmp(user_name, msi_config->users[idx].name) == 0) {
261 			*num_vectors = msi_config->users[idx].num_vectors;
262 			*base_vector =  msi_config->users[idx].base_vector;
263 			*user_base_data = *base_vector + ab->pci.msi.ep_base_data;
264 
265 			ath11k_dbg(ab, ATH11K_DBG_PCI,
266 				   "msi assignment %s num_vectors %d user_base_data %u base_vector %u\n",
267 				   user_name, *num_vectors, *user_base_data,
268 				   *base_vector);
269 
270 			return 0;
271 		}
272 	}
273 
274 	ath11k_err(ab, "Failed to find MSI assignment for %s!\n", user_name);
275 
276 	return -EINVAL;
277 }
278 EXPORT_SYMBOL(ath11k_pcic_get_user_msi_assignment);
279 
280 void ath11k_pcic_get_ce_msi_idx(struct ath11k_base *ab, u32 ce_id, u32 *msi_idx)
281 {
282 	u32 i, msi_data_idx;
283 
284 	for (i = 0, msi_data_idx = 0; i < ab->hw_params.ce_count; i++) {
285 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
286 			continue;
287 
288 		if (ce_id == i)
289 			break;
290 
291 		msi_data_idx++;
292 	}
293 	*msi_idx = msi_data_idx;
294 }
295 EXPORT_SYMBOL(ath11k_pcic_get_ce_msi_idx);
296 
297 static void ath11k_pcic_free_ext_irq(struct ath11k_base *ab)
298 {
299 	int i, j;
300 
301 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
302 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
303 
304 		for (j = 0; j < irq_grp->num_irq; j++)
305 			free_irq(ab->irq_num[irq_grp->irqs[j]], irq_grp);
306 
307 		netif_napi_del(&irq_grp->napi);
308 	}
309 }
310 
311 void ath11k_pcic_free_irq(struct ath11k_base *ab)
312 {
313 	int i, irq_idx;
314 
315 	for (i = 0; i < ab->hw_params.ce_count; i++) {
316 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
317 			continue;
318 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
319 		free_irq(ab->irq_num[irq_idx], &ab->ce.ce_pipe[i]);
320 	}
321 
322 	ath11k_pcic_free_ext_irq(ab);
323 }
324 EXPORT_SYMBOL(ath11k_pcic_free_irq);
325 
326 static void ath11k_pcic_ce_irq_enable(struct ath11k_base *ab, u16 ce_id)
327 {
328 	u32 irq_idx;
329 
330 	/* In case of one MSI vector, we handle irq enable/disable in a
331 	 * uniform way since we only have one irq
332 	 */
333 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
334 		return;
335 
336 	irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_id;
337 	enable_irq(ab->irq_num[irq_idx]);
338 }
339 
340 static void ath11k_pcic_ce_irq_disable(struct ath11k_base *ab, u16 ce_id)
341 {
342 	u32 irq_idx;
343 
344 	/* In case of one MSI vector, we handle irq enable/disable in a
345 	 * uniform way since we only have one irq
346 	 */
347 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
348 		return;
349 
350 	irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_id;
351 	disable_irq_nosync(ab->irq_num[irq_idx]);
352 }
353 
354 static void ath11k_pcic_ce_irqs_disable(struct ath11k_base *ab)
355 {
356 	int i;
357 
358 	clear_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags);
359 
360 	for (i = 0; i < ab->hw_params.ce_count; i++) {
361 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
362 			continue;
363 		ath11k_pcic_ce_irq_disable(ab, i);
364 	}
365 }
366 
367 static void ath11k_pcic_sync_ce_irqs(struct ath11k_base *ab)
368 {
369 	int i;
370 	int irq_idx;
371 
372 	for (i = 0; i < ab->hw_params.ce_count; i++) {
373 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
374 			continue;
375 
376 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
377 		synchronize_irq(ab->irq_num[irq_idx]);
378 	}
379 }
380 
381 static void ath11k_pcic_ce_tasklet(struct tasklet_struct *t)
382 {
383 	struct ath11k_ce_pipe *ce_pipe = from_tasklet(ce_pipe, t, intr_tq);
384 	int irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num;
385 
386 	ath11k_ce_per_engine_service(ce_pipe->ab, ce_pipe->pipe_num);
387 
388 	enable_irq(ce_pipe->ab->irq_num[irq_idx]);
389 }
390 
391 static irqreturn_t ath11k_pcic_ce_interrupt_handler(int irq, void *arg)
392 {
393 	struct ath11k_ce_pipe *ce_pipe = arg;
394 	struct ath11k_base *ab = ce_pipe->ab;
395 	int irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num;
396 
397 	if (!test_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags))
398 		return IRQ_HANDLED;
399 
400 	/* last interrupt received for this CE */
401 	ce_pipe->timestamp = jiffies;
402 
403 	disable_irq_nosync(ab->irq_num[irq_idx]);
404 
405 	tasklet_schedule(&ce_pipe->intr_tq);
406 
407 	return IRQ_HANDLED;
408 }
409 
410 static void ath11k_pcic_ext_grp_disable(struct ath11k_ext_irq_grp *irq_grp)
411 {
412 	struct ath11k_base *ab = irq_grp->ab;
413 	int i;
414 
415 	/* In case of one MSI vector, we handle irq enable/disable
416 	 * in a uniform way since we only have one irq
417 	 */
418 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
419 		return;
420 
421 	for (i = 0; i < irq_grp->num_irq; i++)
422 		disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
423 }
424 
425 static void __ath11k_pcic_ext_irq_disable(struct ath11k_base *sc)
426 {
427 	int i;
428 
429 	clear_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &sc->dev_flags);
430 
431 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
432 		struct ath11k_ext_irq_grp *irq_grp = &sc->ext_irq_grp[i];
433 
434 		ath11k_pcic_ext_grp_disable(irq_grp);
435 
436 		if (irq_grp->napi_enabled) {
437 			napi_synchronize(&irq_grp->napi);
438 			napi_disable(&irq_grp->napi);
439 			irq_grp->napi_enabled = false;
440 		}
441 	}
442 }
443 
444 static void ath11k_pcic_ext_grp_enable(struct ath11k_ext_irq_grp *irq_grp)
445 {
446 	struct ath11k_base *ab = irq_grp->ab;
447 	int i;
448 
449 	/* In case of one MSI vector, we handle irq enable/disable in a
450 	 * uniform way since we only have one irq
451 	 */
452 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
453 		return;
454 
455 	for (i = 0; i < irq_grp->num_irq; i++)
456 		enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
457 }
458 
459 void ath11k_pcic_ext_irq_enable(struct ath11k_base *ab)
460 {
461 	int i;
462 
463 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
464 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
465 
466 		if (!irq_grp->napi_enabled) {
467 			napi_enable(&irq_grp->napi);
468 			irq_grp->napi_enabled = true;
469 		}
470 		ath11k_pcic_ext_grp_enable(irq_grp);
471 	}
472 
473 	set_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags);
474 }
475 EXPORT_SYMBOL(ath11k_pcic_ext_irq_enable);
476 
477 static void ath11k_pcic_sync_ext_irqs(struct ath11k_base *ab)
478 {
479 	int i, j, irq_idx;
480 
481 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
482 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
483 
484 		for (j = 0; j < irq_grp->num_irq; j++) {
485 			irq_idx = irq_grp->irqs[j];
486 			synchronize_irq(ab->irq_num[irq_idx]);
487 		}
488 	}
489 }
490 
491 void ath11k_pcic_ext_irq_disable(struct ath11k_base *ab)
492 {
493 	__ath11k_pcic_ext_irq_disable(ab);
494 	ath11k_pcic_sync_ext_irqs(ab);
495 }
496 EXPORT_SYMBOL(ath11k_pcic_ext_irq_disable);
497 
498 static int ath11k_pcic_ext_grp_napi_poll(struct napi_struct *napi, int budget)
499 {
500 	struct ath11k_ext_irq_grp *irq_grp = container_of(napi,
501 						struct ath11k_ext_irq_grp,
502 						napi);
503 	struct ath11k_base *ab = irq_grp->ab;
504 	int work_done;
505 	int i;
506 
507 	work_done = ath11k_dp_service_srng(ab, irq_grp, budget);
508 	if (work_done < budget) {
509 		napi_complete_done(napi, work_done);
510 		for (i = 0; i < irq_grp->num_irq; i++)
511 			enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
512 	}
513 
514 	if (work_done > budget)
515 		work_done = budget;
516 
517 	return work_done;
518 }
519 
520 static irqreturn_t ath11k_pcic_ext_interrupt_handler(int irq, void *arg)
521 {
522 	struct ath11k_ext_irq_grp *irq_grp = arg;
523 	struct ath11k_base *ab = irq_grp->ab;
524 	int i;
525 
526 	if (!test_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags))
527 		return IRQ_HANDLED;
528 
529 	ath11k_dbg(irq_grp->ab, ATH11K_DBG_PCI, "ext irq %d\n", irq);
530 
531 	/* last interrupt received for this group */
532 	irq_grp->timestamp = jiffies;
533 
534 	for (i = 0; i < irq_grp->num_irq; i++)
535 		disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
536 
537 	napi_schedule(&irq_grp->napi);
538 
539 	return IRQ_HANDLED;
540 }
541 
542 static int
543 ath11k_pcic_get_msi_irq(struct ath11k_base *ab, unsigned int vector)
544 {
545 	return ab->pci.ops->get_msi_irq(ab, vector);
546 }
547 
548 static int ath11k_pcic_ext_irq_config(struct ath11k_base *ab)
549 {
550 	int i, j, ret, num_vectors = 0;
551 	u32 user_base_data = 0, base_vector = 0;
552 	unsigned long irq_flags;
553 
554 	ret = ath11k_pcic_get_user_msi_assignment(ab, "DP", &num_vectors,
555 						  &user_base_data,
556 						  &base_vector);
557 	if (ret < 0)
558 		return ret;
559 
560 	irq_flags = IRQF_SHARED;
561 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
562 		irq_flags |= IRQF_NOBALANCING;
563 
564 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
565 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
566 		u32 num_irq = 0;
567 
568 		irq_grp->ab = ab;
569 		irq_grp->grp_id = i;
570 		init_dummy_netdev(&irq_grp->napi_ndev);
571 		netif_napi_add(&irq_grp->napi_ndev, &irq_grp->napi,
572 			       ath11k_pcic_ext_grp_napi_poll);
573 
574 		if (ab->hw_params.ring_mask->tx[i] ||
575 		    ab->hw_params.ring_mask->rx[i] ||
576 		    ab->hw_params.ring_mask->rx_err[i] ||
577 		    ab->hw_params.ring_mask->rx_wbm_rel[i] ||
578 		    ab->hw_params.ring_mask->reo_status[i] ||
579 		    ab->hw_params.ring_mask->rxdma2host[i] ||
580 		    ab->hw_params.ring_mask->host2rxdma[i] ||
581 		    ab->hw_params.ring_mask->rx_mon_status[i]) {
582 			num_irq = 1;
583 		}
584 
585 		irq_grp->num_irq = num_irq;
586 		irq_grp->irqs[0] = ATH11K_PCI_IRQ_DP_OFFSET + i;
587 
588 		for (j = 0; j < irq_grp->num_irq; j++) {
589 			int irq_idx = irq_grp->irqs[j];
590 			int vector = (i % num_vectors) + base_vector;
591 			int irq = ath11k_pcic_get_msi_irq(ab, vector);
592 
593 			if (irq < 0)
594 				return irq;
595 
596 			ab->irq_num[irq_idx] = irq;
597 
598 			ath11k_dbg(ab, ATH11K_DBG_PCI,
599 				   "irq %d group %d\n", irq, i);
600 
601 			irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
602 			ret = request_irq(irq, ath11k_pcic_ext_interrupt_handler,
603 					  irq_flags, "DP_EXT_IRQ", irq_grp);
604 			if (ret) {
605 				ath11k_err(ab, "failed request irq %d: %d\n",
606 					   vector, ret);
607 				return ret;
608 			}
609 		}
610 		ath11k_pcic_ext_grp_disable(irq_grp);
611 	}
612 
613 	return 0;
614 }
615 
616 int ath11k_pcic_config_irq(struct ath11k_base *ab)
617 {
618 	struct ath11k_ce_pipe *ce_pipe;
619 	u32 msi_data_start;
620 	u32 msi_data_count, msi_data_idx;
621 	u32 msi_irq_start;
622 	unsigned int msi_data;
623 	int irq, i, ret, irq_idx;
624 	unsigned long irq_flags;
625 
626 	ret = ath11k_pcic_get_user_msi_assignment(ab, "CE", &msi_data_count,
627 						  &msi_data_start, &msi_irq_start);
628 	if (ret)
629 		return ret;
630 
631 	irq_flags = IRQF_SHARED;
632 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
633 		irq_flags |= IRQF_NOBALANCING;
634 
635 	/* Configure CE irqs */
636 	for (i = 0, msi_data_idx = 0; i < ab->hw_params.ce_count; i++) {
637 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
638 			continue;
639 
640 		msi_data = (msi_data_idx % msi_data_count) + msi_irq_start;
641 		irq = ath11k_pcic_get_msi_irq(ab, msi_data);
642 		if (irq < 0)
643 			return irq;
644 
645 		ce_pipe = &ab->ce.ce_pipe[i];
646 
647 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
648 
649 		tasklet_setup(&ce_pipe->intr_tq, ath11k_pcic_ce_tasklet);
650 
651 		ret = request_irq(irq, ath11k_pcic_ce_interrupt_handler,
652 				  irq_flags, irq_name[irq_idx], ce_pipe);
653 		if (ret) {
654 			ath11k_err(ab, "failed to request irq %d: %d\n",
655 				   irq_idx, ret);
656 			return ret;
657 		}
658 
659 		ab->irq_num[irq_idx] = irq;
660 		msi_data_idx++;
661 
662 		ath11k_pcic_ce_irq_disable(ab, i);
663 	}
664 
665 	ret = ath11k_pcic_ext_irq_config(ab);
666 	if (ret)
667 		return ret;
668 
669 	return 0;
670 }
671 EXPORT_SYMBOL(ath11k_pcic_config_irq);
672 
673 void ath11k_pcic_ce_irqs_enable(struct ath11k_base *ab)
674 {
675 	int i;
676 
677 	set_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags);
678 
679 	for (i = 0; i < ab->hw_params.ce_count; i++) {
680 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
681 			continue;
682 		ath11k_pcic_ce_irq_enable(ab, i);
683 	}
684 }
685 EXPORT_SYMBOL(ath11k_pcic_ce_irqs_enable);
686 
687 static void ath11k_pcic_kill_tasklets(struct ath11k_base *ab)
688 {
689 	int i;
690 
691 	for (i = 0; i < ab->hw_params.ce_count; i++) {
692 		struct ath11k_ce_pipe *ce_pipe = &ab->ce.ce_pipe[i];
693 
694 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
695 			continue;
696 
697 		tasklet_kill(&ce_pipe->intr_tq);
698 	}
699 }
700 
701 void ath11k_pcic_ce_irq_disable_sync(struct ath11k_base *ab)
702 {
703 	ath11k_pcic_ce_irqs_disable(ab);
704 	ath11k_pcic_sync_ce_irqs(ab);
705 	ath11k_pcic_kill_tasklets(ab);
706 }
707 EXPORT_SYMBOL(ath11k_pcic_ce_irq_disable_sync);
708 
709 void ath11k_pcic_stop(struct ath11k_base *ab)
710 {
711 	ath11k_pcic_ce_irq_disable_sync(ab);
712 	ath11k_ce_cleanup_pipes(ab);
713 }
714 EXPORT_SYMBOL(ath11k_pcic_stop);
715 
716 int ath11k_pcic_start(struct ath11k_base *ab)
717 {
718 	set_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags);
719 
720 	ath11k_pcic_ce_irqs_enable(ab);
721 	ath11k_ce_rx_post_buf(ab);
722 
723 	return 0;
724 }
725 EXPORT_SYMBOL(ath11k_pcic_start);
726 
727 int ath11k_pcic_map_service_to_pipe(struct ath11k_base *ab, u16 service_id,
728 				    u8 *ul_pipe, u8 *dl_pipe)
729 {
730 	const struct service_to_pipe *entry;
731 	bool ul_set = false, dl_set = false;
732 	int i;
733 
734 	for (i = 0; i < ab->hw_params.svc_to_ce_map_len; i++) {
735 		entry = &ab->hw_params.svc_to_ce_map[i];
736 
737 		if (__le32_to_cpu(entry->service_id) != service_id)
738 			continue;
739 
740 		switch (__le32_to_cpu(entry->pipedir)) {
741 		case PIPEDIR_NONE:
742 			break;
743 		case PIPEDIR_IN:
744 			WARN_ON(dl_set);
745 			*dl_pipe = __le32_to_cpu(entry->pipenum);
746 			dl_set = true;
747 			break;
748 		case PIPEDIR_OUT:
749 			WARN_ON(ul_set);
750 			*ul_pipe = __le32_to_cpu(entry->pipenum);
751 			ul_set = true;
752 			break;
753 		case PIPEDIR_INOUT:
754 			WARN_ON(dl_set);
755 			WARN_ON(ul_set);
756 			*dl_pipe = __le32_to_cpu(entry->pipenum);
757 			*ul_pipe = __le32_to_cpu(entry->pipenum);
758 			dl_set = true;
759 			ul_set = true;
760 			break;
761 		}
762 	}
763 
764 	if (WARN_ON(!ul_set || !dl_set))
765 		return -ENOENT;
766 
767 	return 0;
768 }
769 EXPORT_SYMBOL(ath11k_pcic_map_service_to_pipe);
770 
771 int ath11k_pcic_register_pci_ops(struct ath11k_base *ab,
772 				 const struct ath11k_pci_ops *pci_ops)
773 {
774 	if (!pci_ops)
775 		return 0;
776 
777 	/* Return error if mandatory pci_ops callbacks are missing */
778 	if (!pci_ops->get_msi_irq || !pci_ops->window_write32 ||
779 	    !pci_ops->window_read32)
780 		return -EINVAL;
781 
782 	ab->pci.ops = pci_ops;
783 	return 0;
784 }
785 EXPORT_SYMBOL(ath11k_pcic_register_pci_ops);
786 
787 void ath11k_pci_enable_ce_irqs_except_wake_irq(struct ath11k_base *ab)
788 {
789 	int i;
790 
791 	for (i = 0; i < ab->hw_params.ce_count; i++) {
792 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR ||
793 		    i == ATH11K_PCI_CE_WAKE_IRQ)
794 			continue;
795 		ath11k_pcic_ce_irq_enable(ab, i);
796 	}
797 }
798 EXPORT_SYMBOL(ath11k_pci_enable_ce_irqs_except_wake_irq);
799 
800 void ath11k_pci_disable_ce_irqs_except_wake_irq(struct ath11k_base *ab)
801 {
802 	int i;
803 	int irq_idx;
804 	struct ath11k_ce_pipe *ce_pipe;
805 
806 	for (i = 0; i < ab->hw_params.ce_count; i++) {
807 		ce_pipe = &ab->ce.ce_pipe[i];
808 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
809 
810 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR ||
811 		    i == ATH11K_PCI_CE_WAKE_IRQ)
812 			continue;
813 
814 		disable_irq_nosync(ab->irq_num[irq_idx]);
815 		synchronize_irq(ab->irq_num[irq_idx]);
816 		tasklet_kill(&ce_pipe->intr_tq);
817 	}
818 }
819 EXPORT_SYMBOL(ath11k_pci_disable_ce_irqs_except_wake_irq);
820