1 // SPDX-License-Identifier: BSD-3-Clause-Clear
2 /*
3  * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
4  * Copyright (c) 2021-2022, Qualcomm Innovation Center, Inc. All rights reserved.
5  */
6 
7 #include "core.h"
8 #include "pcic.h"
9 #include "debug.h"
10 
11 static const char *irq_name[ATH11K_IRQ_NUM_MAX] = {
12 	"bhi",
13 	"mhi-er0",
14 	"mhi-er1",
15 	"ce0",
16 	"ce1",
17 	"ce2",
18 	"ce3",
19 	"ce4",
20 	"ce5",
21 	"ce6",
22 	"ce7",
23 	"ce8",
24 	"ce9",
25 	"ce10",
26 	"ce11",
27 	"host2wbm-desc-feed",
28 	"host2reo-re-injection",
29 	"host2reo-command",
30 	"host2rxdma-monitor-ring3",
31 	"host2rxdma-monitor-ring2",
32 	"host2rxdma-monitor-ring1",
33 	"reo2ost-exception",
34 	"wbm2host-rx-release",
35 	"reo2host-status",
36 	"reo2host-destination-ring4",
37 	"reo2host-destination-ring3",
38 	"reo2host-destination-ring2",
39 	"reo2host-destination-ring1",
40 	"rxdma2host-monitor-destination-mac3",
41 	"rxdma2host-monitor-destination-mac2",
42 	"rxdma2host-monitor-destination-mac1",
43 	"ppdu-end-interrupts-mac3",
44 	"ppdu-end-interrupts-mac2",
45 	"ppdu-end-interrupts-mac1",
46 	"rxdma2host-monitor-status-ring-mac3",
47 	"rxdma2host-monitor-status-ring-mac2",
48 	"rxdma2host-monitor-status-ring-mac1",
49 	"host2rxdma-host-buf-ring-mac3",
50 	"host2rxdma-host-buf-ring-mac2",
51 	"host2rxdma-host-buf-ring-mac1",
52 	"rxdma2host-destination-ring-mac3",
53 	"rxdma2host-destination-ring-mac2",
54 	"rxdma2host-destination-ring-mac1",
55 	"host2tcl-input-ring4",
56 	"host2tcl-input-ring3",
57 	"host2tcl-input-ring2",
58 	"host2tcl-input-ring1",
59 	"wbm2host-tx-completions-ring3",
60 	"wbm2host-tx-completions-ring2",
61 	"wbm2host-tx-completions-ring1",
62 	"tcl2host-status-ring",
63 };
64 
65 static const struct ath11k_msi_config ath11k_msi_config[] = {
66 	{
67 		.total_vectors = 32,
68 		.total_users = 4,
69 		.users = (struct ath11k_msi_user[]) {
70 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
71 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
72 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
73 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
74 		},
75 		.hw_rev = ATH11K_HW_QCA6390_HW20,
76 	},
77 	{
78 		.total_vectors = 16,
79 		.total_users = 3,
80 		.users = (struct ath11k_msi_user[]) {
81 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
82 			{ .name = "CE", .num_vectors = 5, .base_vector = 3 },
83 			{ .name = "DP", .num_vectors = 8, .base_vector = 8 },
84 		},
85 		.hw_rev = ATH11K_HW_QCN9074_HW10,
86 	},
87 	{
88 		.total_vectors = 32,
89 		.total_users = 4,
90 		.users = (struct ath11k_msi_user[]) {
91 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
92 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
93 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
94 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
95 		},
96 		.hw_rev = ATH11K_HW_WCN6855_HW20,
97 	},
98 	{
99 		.total_vectors = 32,
100 		.total_users = 4,
101 		.users = (struct ath11k_msi_user[]) {
102 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
103 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
104 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
105 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
106 		},
107 		.hw_rev = ATH11K_HW_WCN6855_HW21,
108 	},
109 	{
110 		.total_vectors = 28,
111 		.total_users = 2,
112 		.users = (struct ath11k_msi_user[]) {
113 			{ .name = "CE", .num_vectors = 10, .base_vector = 0 },
114 			{ .name = "DP", .num_vectors = 18, .base_vector = 10 },
115 		},
116 		.hw_rev = ATH11K_HW_WCN6750_HW10,
117 	},
118 };
119 
120 int ath11k_pcic_init_msi_config(struct ath11k_base *ab)
121 {
122 	const struct ath11k_msi_config *msi_config;
123 	int i;
124 
125 	for (i = 0; i < ARRAY_SIZE(ath11k_msi_config); i++) {
126 		msi_config = &ath11k_msi_config[i];
127 
128 		if (msi_config->hw_rev == ab->hw_rev)
129 			break;
130 	}
131 
132 	if (i == ARRAY_SIZE(ath11k_msi_config)) {
133 		ath11k_err(ab, "failed to fetch msi config, unsupported hw version: 0x%x\n",
134 			   ab->hw_rev);
135 		return -EINVAL;
136 	}
137 
138 	ab->pci.msi.config = msi_config;
139 	return 0;
140 }
141 EXPORT_SYMBOL(ath11k_pcic_init_msi_config);
142 
143 static inline u32 ath11k_pcic_get_window_start(struct ath11k_base *ab,
144 					       u32 offset)
145 {
146 	u32 window_start = 0;
147 
148 	if ((offset ^ HAL_SEQ_WCSS_UMAC_OFFSET) < ATH11K_PCI_WINDOW_RANGE_MASK)
149 		window_start = ab->hw_params.dp_window_idx * ATH11K_PCI_WINDOW_START;
150 	else if ((offset ^ HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab)) <
151 		 ATH11K_PCI_WINDOW_RANGE_MASK)
152 		window_start = ab->hw_params.ce_window_idx * ATH11K_PCI_WINDOW_START;
153 
154 	return window_start;
155 }
156 
157 void ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value)
158 {
159 	u32 window_start;
160 	int ret = 0;
161 
162 	/* for offset beyond BAR + 4K - 32, may
163 	 * need to wakeup the device to access.
164 	 */
165 	if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
166 	    offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->wakeup)
167 		ret = ab->pci.ops->wakeup(ab);
168 
169 	if (offset < ATH11K_PCI_WINDOW_START) {
170 		iowrite32(value, ab->mem  + offset);
171 	} else if (ab->hw_params.static_window_map) {
172 		window_start = ath11k_pcic_get_window_start(ab, offset);
173 		iowrite32(value, ab->mem + window_start +
174 			  (offset & ATH11K_PCI_WINDOW_RANGE_MASK));
175 	} else if (ab->pci.ops->window_write32) {
176 		ab->pci.ops->window_write32(ab, offset, value);
177 	}
178 
179 	if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
180 	    offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->release &&
181 	    !ret)
182 		ab->pci.ops->release(ab);
183 }
184 EXPORT_SYMBOL(ath11k_pcic_write32);
185 
186 u32 ath11k_pcic_read32(struct ath11k_base *ab, u32 offset)
187 {
188 	u32 val = 0;
189 	u32 window_start;
190 	int ret = 0;
191 
192 	/* for offset beyond BAR + 4K - 32, may
193 	 * need to wakeup the device to access.
194 	 */
195 	if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
196 	    offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->wakeup)
197 		ret = ab->pci.ops->wakeup(ab);
198 
199 	if (offset < ATH11K_PCI_WINDOW_START) {
200 		val = ioread32(ab->mem + offset);
201 	} else if (ab->hw_params.static_window_map) {
202 		window_start = ath11k_pcic_get_window_start(ab, offset);
203 		val = ioread32(ab->mem + window_start +
204 			       (offset & ATH11K_PCI_WINDOW_RANGE_MASK));
205 	} else if (ab->pci.ops->window_read32) {
206 		val = ab->pci.ops->window_read32(ab, offset);
207 	}
208 
209 	if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
210 	    offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->release &&
211 	    !ret)
212 		ab->pci.ops->release(ab);
213 
214 	return val;
215 }
216 EXPORT_SYMBOL(ath11k_pcic_read32);
217 
218 void ath11k_pcic_get_msi_address(struct ath11k_base *ab, u32 *msi_addr_lo,
219 				 u32 *msi_addr_hi)
220 {
221 	*msi_addr_lo = ab->pci.msi.addr_lo;
222 	*msi_addr_hi = ab->pci.msi.addr_hi;
223 }
224 EXPORT_SYMBOL(ath11k_pcic_get_msi_address);
225 
226 int ath11k_pcic_get_user_msi_assignment(struct ath11k_base *ab, char *user_name,
227 					int *num_vectors, u32 *user_base_data,
228 					u32 *base_vector)
229 {
230 	const struct ath11k_msi_config *msi_config = ab->pci.msi.config;
231 	int idx;
232 
233 	for (idx = 0; idx < msi_config->total_users; idx++) {
234 		if (strcmp(user_name, msi_config->users[idx].name) == 0) {
235 			*num_vectors = msi_config->users[idx].num_vectors;
236 			*base_vector =  msi_config->users[idx].base_vector;
237 			*user_base_data = *base_vector + ab->pci.msi.ep_base_data;
238 
239 			ath11k_dbg(ab, ATH11K_DBG_PCI,
240 				   "Assign MSI to user: %s, num_vectors: %d, user_base_data: %u, base_vector: %u\n",
241 				   user_name, *num_vectors, *user_base_data,
242 				   *base_vector);
243 
244 			return 0;
245 		}
246 	}
247 
248 	ath11k_err(ab, "Failed to find MSI assignment for %s!\n", user_name);
249 
250 	return -EINVAL;
251 }
252 EXPORT_SYMBOL(ath11k_pcic_get_user_msi_assignment);
253 
254 void ath11k_pcic_get_ce_msi_idx(struct ath11k_base *ab, u32 ce_id, u32 *msi_idx)
255 {
256 	u32 i, msi_data_idx;
257 
258 	for (i = 0, msi_data_idx = 0; i < ab->hw_params.ce_count; i++) {
259 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
260 			continue;
261 
262 		if (ce_id == i)
263 			break;
264 
265 		msi_data_idx++;
266 	}
267 	*msi_idx = msi_data_idx;
268 }
269 EXPORT_SYMBOL(ath11k_pcic_get_ce_msi_idx);
270 
271 static void ath11k_pcic_free_ext_irq(struct ath11k_base *ab)
272 {
273 	int i, j;
274 
275 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
276 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
277 
278 		for (j = 0; j < irq_grp->num_irq; j++)
279 			free_irq(ab->irq_num[irq_grp->irqs[j]], irq_grp);
280 
281 		netif_napi_del(&irq_grp->napi);
282 	}
283 }
284 
285 void ath11k_pcic_free_irq(struct ath11k_base *ab)
286 {
287 	int i, irq_idx;
288 
289 	for (i = 0; i < ab->hw_params.ce_count; i++) {
290 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
291 			continue;
292 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
293 		free_irq(ab->irq_num[irq_idx], &ab->ce.ce_pipe[i]);
294 	}
295 
296 	ath11k_pcic_free_ext_irq(ab);
297 }
298 EXPORT_SYMBOL(ath11k_pcic_free_irq);
299 
300 static void ath11k_pcic_ce_irq_enable(struct ath11k_base *ab, u16 ce_id)
301 {
302 	u32 irq_idx;
303 
304 	/* In case of one MSI vector, we handle irq enable/disable in a
305 	 * uniform way since we only have one irq
306 	 */
307 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
308 		return;
309 
310 	irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_id;
311 	enable_irq(ab->irq_num[irq_idx]);
312 }
313 
314 static void ath11k_pcic_ce_irq_disable(struct ath11k_base *ab, u16 ce_id)
315 {
316 	u32 irq_idx;
317 
318 	/* In case of one MSI vector, we handle irq enable/disable in a
319 	 * uniform way since we only have one irq
320 	 */
321 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
322 		return;
323 
324 	irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_id;
325 	disable_irq_nosync(ab->irq_num[irq_idx]);
326 }
327 
328 static void ath11k_pcic_ce_irqs_disable(struct ath11k_base *ab)
329 {
330 	int i;
331 
332 	clear_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags);
333 
334 	for (i = 0; i < ab->hw_params.ce_count; i++) {
335 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
336 			continue;
337 		ath11k_pcic_ce_irq_disable(ab, i);
338 	}
339 }
340 
341 static void ath11k_pcic_sync_ce_irqs(struct ath11k_base *ab)
342 {
343 	int i;
344 	int irq_idx;
345 
346 	for (i = 0; i < ab->hw_params.ce_count; i++) {
347 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
348 			continue;
349 
350 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
351 		synchronize_irq(ab->irq_num[irq_idx]);
352 	}
353 }
354 
355 static void ath11k_pcic_ce_tasklet(struct tasklet_struct *t)
356 {
357 	struct ath11k_ce_pipe *ce_pipe = from_tasklet(ce_pipe, t, intr_tq);
358 	int irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num;
359 
360 	ath11k_ce_per_engine_service(ce_pipe->ab, ce_pipe->pipe_num);
361 
362 	enable_irq(ce_pipe->ab->irq_num[irq_idx]);
363 }
364 
365 static irqreturn_t ath11k_pcic_ce_interrupt_handler(int irq, void *arg)
366 {
367 	struct ath11k_ce_pipe *ce_pipe = arg;
368 	struct ath11k_base *ab = ce_pipe->ab;
369 	int irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num;
370 
371 	if (!test_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags))
372 		return IRQ_HANDLED;
373 
374 	/* last interrupt received for this CE */
375 	ce_pipe->timestamp = jiffies;
376 
377 	disable_irq_nosync(ab->irq_num[irq_idx]);
378 
379 	tasklet_schedule(&ce_pipe->intr_tq);
380 
381 	return IRQ_HANDLED;
382 }
383 
384 static void ath11k_pcic_ext_grp_disable(struct ath11k_ext_irq_grp *irq_grp)
385 {
386 	struct ath11k_base *ab = irq_grp->ab;
387 	int i;
388 
389 	/* In case of one MSI vector, we handle irq enable/disable
390 	 * in a uniform way since we only have one irq
391 	 */
392 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
393 		return;
394 
395 	for (i = 0; i < irq_grp->num_irq; i++)
396 		disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
397 }
398 
399 static void __ath11k_pcic_ext_irq_disable(struct ath11k_base *sc)
400 {
401 	int i;
402 
403 	clear_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &sc->dev_flags);
404 
405 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
406 		struct ath11k_ext_irq_grp *irq_grp = &sc->ext_irq_grp[i];
407 
408 		ath11k_pcic_ext_grp_disable(irq_grp);
409 
410 		if (irq_grp->napi_enabled) {
411 			napi_synchronize(&irq_grp->napi);
412 			napi_disable(&irq_grp->napi);
413 			irq_grp->napi_enabled = false;
414 		}
415 	}
416 }
417 
418 static void ath11k_pcic_ext_grp_enable(struct ath11k_ext_irq_grp *irq_grp)
419 {
420 	struct ath11k_base *ab = irq_grp->ab;
421 	int i;
422 
423 	/* In case of one MSI vector, we handle irq enable/disable in a
424 	 * uniform way since we only have one irq
425 	 */
426 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
427 		return;
428 
429 	for (i = 0; i < irq_grp->num_irq; i++)
430 		enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
431 }
432 
433 void ath11k_pcic_ext_irq_enable(struct ath11k_base *ab)
434 {
435 	int i;
436 
437 	set_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags);
438 
439 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
440 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
441 
442 		if (!irq_grp->napi_enabled) {
443 			napi_enable(&irq_grp->napi);
444 			irq_grp->napi_enabled = true;
445 		}
446 		ath11k_pcic_ext_grp_enable(irq_grp);
447 	}
448 }
449 EXPORT_SYMBOL(ath11k_pcic_ext_irq_enable);
450 
451 static void ath11k_pcic_sync_ext_irqs(struct ath11k_base *ab)
452 {
453 	int i, j, irq_idx;
454 
455 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
456 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
457 
458 		for (j = 0; j < irq_grp->num_irq; j++) {
459 			irq_idx = irq_grp->irqs[j];
460 			synchronize_irq(ab->irq_num[irq_idx]);
461 		}
462 	}
463 }
464 
465 void ath11k_pcic_ext_irq_disable(struct ath11k_base *ab)
466 {
467 	__ath11k_pcic_ext_irq_disable(ab);
468 	ath11k_pcic_sync_ext_irqs(ab);
469 }
470 EXPORT_SYMBOL(ath11k_pcic_ext_irq_disable);
471 
472 static int ath11k_pcic_ext_grp_napi_poll(struct napi_struct *napi, int budget)
473 {
474 	struct ath11k_ext_irq_grp *irq_grp = container_of(napi,
475 						struct ath11k_ext_irq_grp,
476 						napi);
477 	struct ath11k_base *ab = irq_grp->ab;
478 	int work_done;
479 	int i;
480 
481 	work_done = ath11k_dp_service_srng(ab, irq_grp, budget);
482 	if (work_done < budget) {
483 		napi_complete_done(napi, work_done);
484 		for (i = 0; i < irq_grp->num_irq; i++)
485 			enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
486 	}
487 
488 	if (work_done > budget)
489 		work_done = budget;
490 
491 	return work_done;
492 }
493 
494 static irqreturn_t ath11k_pcic_ext_interrupt_handler(int irq, void *arg)
495 {
496 	struct ath11k_ext_irq_grp *irq_grp = arg;
497 	struct ath11k_base *ab = irq_grp->ab;
498 	int i;
499 
500 	if (!test_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags))
501 		return IRQ_HANDLED;
502 
503 	ath11k_dbg(irq_grp->ab, ATH11K_DBG_PCI, "ext irq:%d\n", irq);
504 
505 	/* last interrupt received for this group */
506 	irq_grp->timestamp = jiffies;
507 
508 	for (i = 0; i < irq_grp->num_irq; i++)
509 		disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
510 
511 	napi_schedule(&irq_grp->napi);
512 
513 	return IRQ_HANDLED;
514 }
515 
516 static int
517 ath11k_pcic_get_msi_irq(struct ath11k_base *ab, unsigned int vector)
518 {
519 	if (!ab->pci.ops->get_msi_irq) {
520 		WARN_ONCE(1, "get_msi_irq pci op not defined");
521 		return -EOPNOTSUPP;
522 	}
523 
524 	return ab->pci.ops->get_msi_irq(ab, vector);
525 }
526 
527 static int ath11k_pcic_ext_irq_config(struct ath11k_base *ab)
528 {
529 	int i, j, ret, num_vectors = 0;
530 	u32 user_base_data = 0, base_vector = 0;
531 	unsigned long irq_flags;
532 
533 	ret = ath11k_pcic_get_user_msi_assignment(ab, "DP", &num_vectors,
534 						  &user_base_data,
535 						  &base_vector);
536 	if (ret < 0)
537 		return ret;
538 
539 	irq_flags = IRQF_SHARED;
540 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
541 		irq_flags |= IRQF_NOBALANCING;
542 
543 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
544 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
545 		u32 num_irq = 0;
546 
547 		irq_grp->ab = ab;
548 		irq_grp->grp_id = i;
549 		init_dummy_netdev(&irq_grp->napi_ndev);
550 		netif_napi_add(&irq_grp->napi_ndev, &irq_grp->napi,
551 			       ath11k_pcic_ext_grp_napi_poll, NAPI_POLL_WEIGHT);
552 
553 		if (ab->hw_params.ring_mask->tx[i] ||
554 		    ab->hw_params.ring_mask->rx[i] ||
555 		    ab->hw_params.ring_mask->rx_err[i] ||
556 		    ab->hw_params.ring_mask->rx_wbm_rel[i] ||
557 		    ab->hw_params.ring_mask->reo_status[i] ||
558 		    ab->hw_params.ring_mask->rxdma2host[i] ||
559 		    ab->hw_params.ring_mask->host2rxdma[i] ||
560 		    ab->hw_params.ring_mask->rx_mon_status[i]) {
561 			num_irq = 1;
562 		}
563 
564 		irq_grp->num_irq = num_irq;
565 		irq_grp->irqs[0] = ATH11K_PCI_IRQ_DP_OFFSET + i;
566 
567 		for (j = 0; j < irq_grp->num_irq; j++) {
568 			int irq_idx = irq_grp->irqs[j];
569 			int vector = (i % num_vectors) + base_vector;
570 			int irq = ath11k_pcic_get_msi_irq(ab, vector);
571 
572 			if (irq < 0)
573 				return irq;
574 
575 			ab->irq_num[irq_idx] = irq;
576 
577 			ath11k_dbg(ab, ATH11K_DBG_PCI,
578 				   "irq:%d group:%d\n", irq, i);
579 
580 			irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
581 			ret = request_irq(irq, ath11k_pcic_ext_interrupt_handler,
582 					  irq_flags, "DP_EXT_IRQ", irq_grp);
583 			if (ret) {
584 				ath11k_err(ab, "failed request irq %d: %d\n",
585 					   vector, ret);
586 				return ret;
587 			}
588 		}
589 		ath11k_pcic_ext_grp_disable(irq_grp);
590 	}
591 
592 	return 0;
593 }
594 
595 int ath11k_pcic_config_irq(struct ath11k_base *ab)
596 {
597 	struct ath11k_ce_pipe *ce_pipe;
598 	u32 msi_data_start;
599 	u32 msi_data_count, msi_data_idx;
600 	u32 msi_irq_start;
601 	unsigned int msi_data;
602 	int irq, i, ret, irq_idx;
603 	unsigned long irq_flags;
604 
605 	ret = ath11k_pcic_get_user_msi_assignment(ab, "CE", &msi_data_count,
606 						  &msi_data_start, &msi_irq_start);
607 	if (ret)
608 		return ret;
609 
610 	irq_flags = IRQF_SHARED;
611 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
612 		irq_flags |= IRQF_NOBALANCING;
613 
614 	/* Configure CE irqs */
615 	for (i = 0, msi_data_idx = 0; i < ab->hw_params.ce_count; i++) {
616 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
617 			continue;
618 
619 		msi_data = (msi_data_idx % msi_data_count) + msi_irq_start;
620 		irq = ath11k_pcic_get_msi_irq(ab, msi_data);
621 		if (irq < 0)
622 			return irq;
623 
624 		ce_pipe = &ab->ce.ce_pipe[i];
625 
626 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
627 
628 		tasklet_setup(&ce_pipe->intr_tq, ath11k_pcic_ce_tasklet);
629 
630 		ret = request_irq(irq, ath11k_pcic_ce_interrupt_handler,
631 				  irq_flags, irq_name[irq_idx], ce_pipe);
632 		if (ret) {
633 			ath11k_err(ab, "failed to request irq %d: %d\n",
634 				   irq_idx, ret);
635 			return ret;
636 		}
637 
638 		ab->irq_num[irq_idx] = irq;
639 		msi_data_idx++;
640 
641 		ath11k_pcic_ce_irq_disable(ab, i);
642 	}
643 
644 	ret = ath11k_pcic_ext_irq_config(ab);
645 	if (ret)
646 		return ret;
647 
648 	return 0;
649 }
650 EXPORT_SYMBOL(ath11k_pcic_config_irq);
651 
652 void ath11k_pcic_ce_irqs_enable(struct ath11k_base *ab)
653 {
654 	int i;
655 
656 	set_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags);
657 
658 	for (i = 0; i < ab->hw_params.ce_count; i++) {
659 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
660 			continue;
661 		ath11k_pcic_ce_irq_enable(ab, i);
662 	}
663 }
664 EXPORT_SYMBOL(ath11k_pcic_ce_irqs_enable);
665 
666 static void ath11k_pcic_kill_tasklets(struct ath11k_base *ab)
667 {
668 	int i;
669 
670 	for (i = 0; i < ab->hw_params.ce_count; i++) {
671 		struct ath11k_ce_pipe *ce_pipe = &ab->ce.ce_pipe[i];
672 
673 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
674 			continue;
675 
676 		tasklet_kill(&ce_pipe->intr_tq);
677 	}
678 }
679 
680 void ath11k_pcic_ce_irq_disable_sync(struct ath11k_base *ab)
681 {
682 	ath11k_pcic_ce_irqs_disable(ab);
683 	ath11k_pcic_sync_ce_irqs(ab);
684 	ath11k_pcic_kill_tasklets(ab);
685 }
686 EXPORT_SYMBOL(ath11k_pcic_ce_irq_disable_sync);
687 
688 void ath11k_pcic_stop(struct ath11k_base *ab)
689 {
690 	ath11k_pcic_ce_irq_disable_sync(ab);
691 	ath11k_ce_cleanup_pipes(ab);
692 }
693 EXPORT_SYMBOL(ath11k_pcic_stop);
694 
695 int ath11k_pcic_start(struct ath11k_base *ab)
696 {
697 	set_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags);
698 
699 	ath11k_pcic_ce_irqs_enable(ab);
700 	ath11k_ce_rx_post_buf(ab);
701 
702 	return 0;
703 }
704 EXPORT_SYMBOL(ath11k_pcic_start);
705 
706 int ath11k_pcic_map_service_to_pipe(struct ath11k_base *ab, u16 service_id,
707 				    u8 *ul_pipe, u8 *dl_pipe)
708 {
709 	const struct service_to_pipe *entry;
710 	bool ul_set = false, dl_set = false;
711 	int i;
712 
713 	for (i = 0; i < ab->hw_params.svc_to_ce_map_len; i++) {
714 		entry = &ab->hw_params.svc_to_ce_map[i];
715 
716 		if (__le32_to_cpu(entry->service_id) != service_id)
717 			continue;
718 
719 		switch (__le32_to_cpu(entry->pipedir)) {
720 		case PIPEDIR_NONE:
721 			break;
722 		case PIPEDIR_IN:
723 			WARN_ON(dl_set);
724 			*dl_pipe = __le32_to_cpu(entry->pipenum);
725 			dl_set = true;
726 			break;
727 		case PIPEDIR_OUT:
728 			WARN_ON(ul_set);
729 			*ul_pipe = __le32_to_cpu(entry->pipenum);
730 			ul_set = true;
731 			break;
732 		case PIPEDIR_INOUT:
733 			WARN_ON(dl_set);
734 			WARN_ON(ul_set);
735 			*dl_pipe = __le32_to_cpu(entry->pipenum);
736 			*ul_pipe = __le32_to_cpu(entry->pipenum);
737 			dl_set = true;
738 			ul_set = true;
739 			break;
740 		}
741 	}
742 
743 	if (WARN_ON(!ul_set || !dl_set))
744 		return -ENOENT;
745 
746 	return 0;
747 }
748 EXPORT_SYMBOL(ath11k_pcic_map_service_to_pipe);
749