1 // SPDX-License-Identifier: BSD-3-Clause-Clear
2 /*
3  * Copyright (c) 2019-2021 The Linux Foundation. All rights reserved.
4  * Copyright (c) 2021-2022, Qualcomm Innovation Center, Inc. All rights reserved.
5  */
6 
7 #include "core.h"
8 #include "pcic.h"
9 #include "debug.h"
10 
11 static const char *irq_name[ATH11K_IRQ_NUM_MAX] = {
12 	"bhi",
13 	"mhi-er0",
14 	"mhi-er1",
15 	"ce0",
16 	"ce1",
17 	"ce2",
18 	"ce3",
19 	"ce4",
20 	"ce5",
21 	"ce6",
22 	"ce7",
23 	"ce8",
24 	"ce9",
25 	"ce10",
26 	"ce11",
27 	"host2wbm-desc-feed",
28 	"host2reo-re-injection",
29 	"host2reo-command",
30 	"host2rxdma-monitor-ring3",
31 	"host2rxdma-monitor-ring2",
32 	"host2rxdma-monitor-ring1",
33 	"reo2ost-exception",
34 	"wbm2host-rx-release",
35 	"reo2host-status",
36 	"reo2host-destination-ring4",
37 	"reo2host-destination-ring3",
38 	"reo2host-destination-ring2",
39 	"reo2host-destination-ring1",
40 	"rxdma2host-monitor-destination-mac3",
41 	"rxdma2host-monitor-destination-mac2",
42 	"rxdma2host-monitor-destination-mac1",
43 	"ppdu-end-interrupts-mac3",
44 	"ppdu-end-interrupts-mac2",
45 	"ppdu-end-interrupts-mac1",
46 	"rxdma2host-monitor-status-ring-mac3",
47 	"rxdma2host-monitor-status-ring-mac2",
48 	"rxdma2host-monitor-status-ring-mac1",
49 	"host2rxdma-host-buf-ring-mac3",
50 	"host2rxdma-host-buf-ring-mac2",
51 	"host2rxdma-host-buf-ring-mac1",
52 	"rxdma2host-destination-ring-mac3",
53 	"rxdma2host-destination-ring-mac2",
54 	"rxdma2host-destination-ring-mac1",
55 	"host2tcl-input-ring4",
56 	"host2tcl-input-ring3",
57 	"host2tcl-input-ring2",
58 	"host2tcl-input-ring1",
59 	"wbm2host-tx-completions-ring3",
60 	"wbm2host-tx-completions-ring2",
61 	"wbm2host-tx-completions-ring1",
62 	"tcl2host-status-ring",
63 };
64 
65 static const struct ath11k_msi_config ath11k_msi_config[] = {
66 	{
67 		.total_vectors = 32,
68 		.total_users = 4,
69 		.users = (struct ath11k_msi_user[]) {
70 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
71 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
72 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
73 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
74 		},
75 		.hw_rev = ATH11K_HW_QCA6390_HW20,
76 	},
77 	{
78 		.total_vectors = 16,
79 		.total_users = 3,
80 		.users = (struct ath11k_msi_user[]) {
81 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
82 			{ .name = "CE", .num_vectors = 5, .base_vector = 3 },
83 			{ .name = "DP", .num_vectors = 8, .base_vector = 8 },
84 		},
85 		.hw_rev = ATH11K_HW_QCN9074_HW10,
86 	},
87 	{
88 		.total_vectors = 32,
89 		.total_users = 4,
90 		.users = (struct ath11k_msi_user[]) {
91 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
92 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
93 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
94 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
95 		},
96 		.hw_rev = ATH11K_HW_WCN6855_HW20,
97 	},
98 	{
99 		.total_vectors = 32,
100 		.total_users = 4,
101 		.users = (struct ath11k_msi_user[]) {
102 			{ .name = "MHI", .num_vectors = 3, .base_vector = 0 },
103 			{ .name = "CE", .num_vectors = 10, .base_vector = 3 },
104 			{ .name = "WAKE", .num_vectors = 1, .base_vector = 13 },
105 			{ .name = "DP", .num_vectors = 18, .base_vector = 14 },
106 		},
107 		.hw_rev = ATH11K_HW_WCN6855_HW21,
108 	},
109 	{
110 		.total_vectors = 28,
111 		.total_users = 2,
112 		.users = (struct ath11k_msi_user[]) {
113 			{ .name = "CE", .num_vectors = 10, .base_vector = 0 },
114 			{ .name = "DP", .num_vectors = 18, .base_vector = 10 },
115 		},
116 		.hw_rev = ATH11K_HW_WCN6750_HW10,
117 	},
118 };
119 
120 int ath11k_pcic_init_msi_config(struct ath11k_base *ab)
121 {
122 	const struct ath11k_msi_config *msi_config;
123 	int i;
124 
125 	for (i = 0; i < ARRAY_SIZE(ath11k_msi_config); i++) {
126 		msi_config = &ath11k_msi_config[i];
127 
128 		if (msi_config->hw_rev == ab->hw_rev)
129 			break;
130 	}
131 
132 	if (i == ARRAY_SIZE(ath11k_msi_config)) {
133 		ath11k_err(ab, "failed to fetch msi config, unsupported hw version: 0x%x\n",
134 			   ab->hw_rev);
135 		return -EINVAL;
136 	}
137 
138 	ab->pci.msi.config = msi_config;
139 	return 0;
140 }
141 EXPORT_SYMBOL(ath11k_pcic_init_msi_config);
142 
143 static void __ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value)
144 {
145 	if (offset < ATH11K_PCI_WINDOW_START)
146 		iowrite32(value, ab->mem  + offset);
147 	else
148 		ab->pci.ops->window_write32(ab, offset, value);
149 }
150 
151 void ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value)
152 {
153 	int ret = 0;
154 	bool wakeup_required;
155 
156 	/* for offset beyond BAR + 4K - 32, may
157 	 * need to wakeup the device to access.
158 	 */
159 	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
160 			  offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
161 	if (wakeup_required && ab->pci.ops->wakeup)
162 		ret = ab->pci.ops->wakeup(ab);
163 
164 	__ath11k_pcic_write32(ab, offset, value);
165 
166 	if (wakeup_required && !ret && ab->pci.ops->release)
167 		ab->pci.ops->release(ab);
168 }
169 EXPORT_SYMBOL(ath11k_pcic_write32);
170 
171 static u32 __ath11k_pcic_read32(struct ath11k_base *ab, u32 offset)
172 {
173 	u32 val;
174 
175 	if (offset < ATH11K_PCI_WINDOW_START)
176 		val = ioread32(ab->mem + offset);
177 	else
178 		val = ab->pci.ops->window_read32(ab, offset);
179 
180 	return val;
181 }
182 
183 u32 ath11k_pcic_read32(struct ath11k_base *ab, u32 offset)
184 {
185 	int ret = 0;
186 	u32 val;
187 	bool wakeup_required;
188 
189 	/* for offset beyond BAR + 4K - 32, may
190 	 * need to wakeup the device to access.
191 	 */
192 	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
193 			  offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
194 	if (wakeup_required && ab->pci.ops->wakeup)
195 		ret = ab->pci.ops->wakeup(ab);
196 
197 	val = __ath11k_pcic_read32(ab, offset);
198 
199 	if (wakeup_required && !ret && ab->pci.ops->release)
200 		ab->pci.ops->release(ab);
201 
202 	return val;
203 }
204 EXPORT_SYMBOL(ath11k_pcic_read32);
205 
206 int ath11k_pcic_read(struct ath11k_base *ab, void *buf, u32 start, u32 end)
207 {
208 	int ret = 0;
209 	bool wakeup_required;
210 	u32 *data = buf;
211 	u32 i;
212 
213 	/* for offset beyond BAR + 4K - 32, may
214 	 * need to wakeup the device to access.
215 	 */
216 	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
217 			  end >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
218 	if (wakeup_required && ab->pci.ops->wakeup) {
219 		ret = ab->pci.ops->wakeup(ab);
220 		if (ret) {
221 			ath11k_warn(ab, "failed to wakeup for read from 0x%x: %d\n",
222 				    start, ret);
223 			return ret;
224 		}
225 	}
226 
227 	for (i = start; i < end + 1; i += 4)
228 		*data++ = __ath11k_pcic_read32(ab, i);
229 
230 	if (wakeup_required && ab->pci.ops->release)
231 		ab->pci.ops->release(ab);
232 
233 	return 0;
234 }
235 EXPORT_SYMBOL(ath11k_pcic_read);
236 
237 void ath11k_pcic_get_msi_address(struct ath11k_base *ab, u32 *msi_addr_lo,
238 				 u32 *msi_addr_hi)
239 {
240 	*msi_addr_lo = ab->pci.msi.addr_lo;
241 	*msi_addr_hi = ab->pci.msi.addr_hi;
242 }
243 EXPORT_SYMBOL(ath11k_pcic_get_msi_address);
244 
245 int ath11k_pcic_get_user_msi_assignment(struct ath11k_base *ab, char *user_name,
246 					int *num_vectors, u32 *user_base_data,
247 					u32 *base_vector)
248 {
249 	const struct ath11k_msi_config *msi_config = ab->pci.msi.config;
250 	int idx;
251 
252 	for (idx = 0; idx < msi_config->total_users; idx++) {
253 		if (strcmp(user_name, msi_config->users[idx].name) == 0) {
254 			*num_vectors = msi_config->users[idx].num_vectors;
255 			*base_vector =  msi_config->users[idx].base_vector;
256 			*user_base_data = *base_vector + ab->pci.msi.ep_base_data;
257 
258 			ath11k_dbg(ab, ATH11K_DBG_PCI,
259 				   "Assign MSI to user: %s, num_vectors: %d, user_base_data: %u, base_vector: %u\n",
260 				   user_name, *num_vectors, *user_base_data,
261 				   *base_vector);
262 
263 			return 0;
264 		}
265 	}
266 
267 	ath11k_err(ab, "Failed to find MSI assignment for %s!\n", user_name);
268 
269 	return -EINVAL;
270 }
271 EXPORT_SYMBOL(ath11k_pcic_get_user_msi_assignment);
272 
273 void ath11k_pcic_get_ce_msi_idx(struct ath11k_base *ab, u32 ce_id, u32 *msi_idx)
274 {
275 	u32 i, msi_data_idx;
276 
277 	for (i = 0, msi_data_idx = 0; i < ab->hw_params.ce_count; i++) {
278 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
279 			continue;
280 
281 		if (ce_id == i)
282 			break;
283 
284 		msi_data_idx++;
285 	}
286 	*msi_idx = msi_data_idx;
287 }
288 EXPORT_SYMBOL(ath11k_pcic_get_ce_msi_idx);
289 
290 static void ath11k_pcic_free_ext_irq(struct ath11k_base *ab)
291 {
292 	int i, j;
293 
294 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
295 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
296 
297 		for (j = 0; j < irq_grp->num_irq; j++)
298 			free_irq(ab->irq_num[irq_grp->irqs[j]], irq_grp);
299 
300 		netif_napi_del(&irq_grp->napi);
301 	}
302 }
303 
304 void ath11k_pcic_free_irq(struct ath11k_base *ab)
305 {
306 	int i, irq_idx;
307 
308 	for (i = 0; i < ab->hw_params.ce_count; i++) {
309 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
310 			continue;
311 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
312 		free_irq(ab->irq_num[irq_idx], &ab->ce.ce_pipe[i]);
313 	}
314 
315 	ath11k_pcic_free_ext_irq(ab);
316 }
317 EXPORT_SYMBOL(ath11k_pcic_free_irq);
318 
319 static void ath11k_pcic_ce_irq_enable(struct ath11k_base *ab, u16 ce_id)
320 {
321 	u32 irq_idx;
322 
323 	/* In case of one MSI vector, we handle irq enable/disable in a
324 	 * uniform way since we only have one irq
325 	 */
326 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
327 		return;
328 
329 	irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_id;
330 	enable_irq(ab->irq_num[irq_idx]);
331 }
332 
333 static void ath11k_pcic_ce_irq_disable(struct ath11k_base *ab, u16 ce_id)
334 {
335 	u32 irq_idx;
336 
337 	/* In case of one MSI vector, we handle irq enable/disable in a
338 	 * uniform way since we only have one irq
339 	 */
340 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
341 		return;
342 
343 	irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_id;
344 	disable_irq_nosync(ab->irq_num[irq_idx]);
345 }
346 
347 static void ath11k_pcic_ce_irqs_disable(struct ath11k_base *ab)
348 {
349 	int i;
350 
351 	clear_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags);
352 
353 	for (i = 0; i < ab->hw_params.ce_count; i++) {
354 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
355 			continue;
356 		ath11k_pcic_ce_irq_disable(ab, i);
357 	}
358 }
359 
360 static void ath11k_pcic_sync_ce_irqs(struct ath11k_base *ab)
361 {
362 	int i;
363 	int irq_idx;
364 
365 	for (i = 0; i < ab->hw_params.ce_count; i++) {
366 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
367 			continue;
368 
369 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
370 		synchronize_irq(ab->irq_num[irq_idx]);
371 	}
372 }
373 
374 static void ath11k_pcic_ce_tasklet(struct tasklet_struct *t)
375 {
376 	struct ath11k_ce_pipe *ce_pipe = from_tasklet(ce_pipe, t, intr_tq);
377 	int irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num;
378 
379 	ath11k_ce_per_engine_service(ce_pipe->ab, ce_pipe->pipe_num);
380 
381 	enable_irq(ce_pipe->ab->irq_num[irq_idx]);
382 }
383 
384 static irqreturn_t ath11k_pcic_ce_interrupt_handler(int irq, void *arg)
385 {
386 	struct ath11k_ce_pipe *ce_pipe = arg;
387 	struct ath11k_base *ab = ce_pipe->ab;
388 	int irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + ce_pipe->pipe_num;
389 
390 	if (!test_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags))
391 		return IRQ_HANDLED;
392 
393 	/* last interrupt received for this CE */
394 	ce_pipe->timestamp = jiffies;
395 
396 	disable_irq_nosync(ab->irq_num[irq_idx]);
397 
398 	tasklet_schedule(&ce_pipe->intr_tq);
399 
400 	return IRQ_HANDLED;
401 }
402 
403 static void ath11k_pcic_ext_grp_disable(struct ath11k_ext_irq_grp *irq_grp)
404 {
405 	struct ath11k_base *ab = irq_grp->ab;
406 	int i;
407 
408 	/* In case of one MSI vector, we handle irq enable/disable
409 	 * in a uniform way since we only have one irq
410 	 */
411 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
412 		return;
413 
414 	for (i = 0; i < irq_grp->num_irq; i++)
415 		disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
416 }
417 
418 static void __ath11k_pcic_ext_irq_disable(struct ath11k_base *sc)
419 {
420 	int i;
421 
422 	clear_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &sc->dev_flags);
423 
424 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
425 		struct ath11k_ext_irq_grp *irq_grp = &sc->ext_irq_grp[i];
426 
427 		ath11k_pcic_ext_grp_disable(irq_grp);
428 
429 		if (irq_grp->napi_enabled) {
430 			napi_synchronize(&irq_grp->napi);
431 			napi_disable(&irq_grp->napi);
432 			irq_grp->napi_enabled = false;
433 		}
434 	}
435 }
436 
437 static void ath11k_pcic_ext_grp_enable(struct ath11k_ext_irq_grp *irq_grp)
438 {
439 	struct ath11k_base *ab = irq_grp->ab;
440 	int i;
441 
442 	/* In case of one MSI vector, we handle irq enable/disable in a
443 	 * uniform way since we only have one irq
444 	 */
445 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
446 		return;
447 
448 	for (i = 0; i < irq_grp->num_irq; i++)
449 		enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
450 }
451 
452 void ath11k_pcic_ext_irq_enable(struct ath11k_base *ab)
453 {
454 	int i;
455 
456 	set_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags);
457 
458 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
459 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
460 
461 		if (!irq_grp->napi_enabled) {
462 			dev_set_threaded(&irq_grp->napi_ndev, true);
463 			napi_enable(&irq_grp->napi);
464 			irq_grp->napi_enabled = true;
465 		}
466 		ath11k_pcic_ext_grp_enable(irq_grp);
467 	}
468 }
469 EXPORT_SYMBOL(ath11k_pcic_ext_irq_enable);
470 
471 static void ath11k_pcic_sync_ext_irqs(struct ath11k_base *ab)
472 {
473 	int i, j, irq_idx;
474 
475 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
476 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
477 
478 		for (j = 0; j < irq_grp->num_irq; j++) {
479 			irq_idx = irq_grp->irqs[j];
480 			synchronize_irq(ab->irq_num[irq_idx]);
481 		}
482 	}
483 }
484 
485 void ath11k_pcic_ext_irq_disable(struct ath11k_base *ab)
486 {
487 	__ath11k_pcic_ext_irq_disable(ab);
488 	ath11k_pcic_sync_ext_irqs(ab);
489 }
490 EXPORT_SYMBOL(ath11k_pcic_ext_irq_disable);
491 
492 static int ath11k_pcic_ext_grp_napi_poll(struct napi_struct *napi, int budget)
493 {
494 	struct ath11k_ext_irq_grp *irq_grp = container_of(napi,
495 						struct ath11k_ext_irq_grp,
496 						napi);
497 	struct ath11k_base *ab = irq_grp->ab;
498 	int work_done;
499 	int i;
500 
501 	work_done = ath11k_dp_service_srng(ab, irq_grp, budget);
502 	if (work_done < budget) {
503 		napi_complete_done(napi, work_done);
504 		for (i = 0; i < irq_grp->num_irq; i++)
505 			enable_irq(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
506 	}
507 
508 	if (work_done > budget)
509 		work_done = budget;
510 
511 	return work_done;
512 }
513 
514 static irqreturn_t ath11k_pcic_ext_interrupt_handler(int irq, void *arg)
515 {
516 	struct ath11k_ext_irq_grp *irq_grp = arg;
517 	struct ath11k_base *ab = irq_grp->ab;
518 	int i;
519 
520 	if (!test_bit(ATH11K_FLAG_EXT_IRQ_ENABLED, &ab->dev_flags))
521 		return IRQ_HANDLED;
522 
523 	ath11k_dbg(irq_grp->ab, ATH11K_DBG_PCI, "ext irq:%d\n", irq);
524 
525 	/* last interrupt received for this group */
526 	irq_grp->timestamp = jiffies;
527 
528 	for (i = 0; i < irq_grp->num_irq; i++)
529 		disable_irq_nosync(irq_grp->ab->irq_num[irq_grp->irqs[i]]);
530 
531 	napi_schedule(&irq_grp->napi);
532 
533 	return IRQ_HANDLED;
534 }
535 
536 static int
537 ath11k_pcic_get_msi_irq(struct ath11k_base *ab, unsigned int vector)
538 {
539 	return ab->pci.ops->get_msi_irq(ab, vector);
540 }
541 
542 static int ath11k_pcic_ext_irq_config(struct ath11k_base *ab)
543 {
544 	int i, j, ret, num_vectors = 0;
545 	u32 user_base_data = 0, base_vector = 0;
546 	unsigned long irq_flags;
547 
548 	ret = ath11k_pcic_get_user_msi_assignment(ab, "DP", &num_vectors,
549 						  &user_base_data,
550 						  &base_vector);
551 	if (ret < 0)
552 		return ret;
553 
554 	irq_flags = IRQF_SHARED;
555 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
556 		irq_flags |= IRQF_NOBALANCING;
557 
558 	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
559 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
560 		u32 num_irq = 0;
561 
562 		irq_grp->ab = ab;
563 		irq_grp->grp_id = i;
564 		init_dummy_netdev(&irq_grp->napi_ndev);
565 		netif_napi_add(&irq_grp->napi_ndev, &irq_grp->napi,
566 			       ath11k_pcic_ext_grp_napi_poll);
567 
568 		if (ab->hw_params.ring_mask->tx[i] ||
569 		    ab->hw_params.ring_mask->rx[i] ||
570 		    ab->hw_params.ring_mask->rx_err[i] ||
571 		    ab->hw_params.ring_mask->rx_wbm_rel[i] ||
572 		    ab->hw_params.ring_mask->reo_status[i] ||
573 		    ab->hw_params.ring_mask->rxdma2host[i] ||
574 		    ab->hw_params.ring_mask->host2rxdma[i] ||
575 		    ab->hw_params.ring_mask->rx_mon_status[i]) {
576 			num_irq = 1;
577 		}
578 
579 		irq_grp->num_irq = num_irq;
580 		irq_grp->irqs[0] = ATH11K_PCI_IRQ_DP_OFFSET + i;
581 
582 		for (j = 0; j < irq_grp->num_irq; j++) {
583 			int irq_idx = irq_grp->irqs[j];
584 			int vector = (i % num_vectors) + base_vector;
585 			int irq = ath11k_pcic_get_msi_irq(ab, vector);
586 
587 			if (irq < 0)
588 				return irq;
589 
590 			ab->irq_num[irq_idx] = irq;
591 
592 			ath11k_dbg(ab, ATH11K_DBG_PCI,
593 				   "irq:%d group:%d\n", irq, i);
594 
595 			irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
596 			ret = request_irq(irq, ath11k_pcic_ext_interrupt_handler,
597 					  irq_flags, "DP_EXT_IRQ", irq_grp);
598 			if (ret) {
599 				ath11k_err(ab, "failed request irq %d: %d\n",
600 					   vector, ret);
601 				return ret;
602 			}
603 		}
604 		ath11k_pcic_ext_grp_disable(irq_grp);
605 	}
606 
607 	return 0;
608 }
609 
610 int ath11k_pcic_config_irq(struct ath11k_base *ab)
611 {
612 	struct ath11k_ce_pipe *ce_pipe;
613 	u32 msi_data_start;
614 	u32 msi_data_count, msi_data_idx;
615 	u32 msi_irq_start;
616 	unsigned int msi_data;
617 	int irq, i, ret, irq_idx;
618 	unsigned long irq_flags;
619 
620 	ret = ath11k_pcic_get_user_msi_assignment(ab, "CE", &msi_data_count,
621 						  &msi_data_start, &msi_irq_start);
622 	if (ret)
623 		return ret;
624 
625 	irq_flags = IRQF_SHARED;
626 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
627 		irq_flags |= IRQF_NOBALANCING;
628 
629 	/* Configure CE irqs */
630 	for (i = 0, msi_data_idx = 0; i < ab->hw_params.ce_count; i++) {
631 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
632 			continue;
633 
634 		msi_data = (msi_data_idx % msi_data_count) + msi_irq_start;
635 		irq = ath11k_pcic_get_msi_irq(ab, msi_data);
636 		if (irq < 0)
637 			return irq;
638 
639 		ce_pipe = &ab->ce.ce_pipe[i];
640 
641 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
642 
643 		tasklet_setup(&ce_pipe->intr_tq, ath11k_pcic_ce_tasklet);
644 
645 		ret = request_irq(irq, ath11k_pcic_ce_interrupt_handler,
646 				  irq_flags, irq_name[irq_idx], ce_pipe);
647 		if (ret) {
648 			ath11k_err(ab, "failed to request irq %d: %d\n",
649 				   irq_idx, ret);
650 			return ret;
651 		}
652 
653 		ab->irq_num[irq_idx] = irq;
654 		msi_data_idx++;
655 
656 		ath11k_pcic_ce_irq_disable(ab, i);
657 	}
658 
659 	ret = ath11k_pcic_ext_irq_config(ab);
660 	if (ret)
661 		return ret;
662 
663 	return 0;
664 }
665 EXPORT_SYMBOL(ath11k_pcic_config_irq);
666 
667 void ath11k_pcic_ce_irqs_enable(struct ath11k_base *ab)
668 {
669 	int i;
670 
671 	set_bit(ATH11K_FLAG_CE_IRQ_ENABLED, &ab->dev_flags);
672 
673 	for (i = 0; i < ab->hw_params.ce_count; i++) {
674 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
675 			continue;
676 		ath11k_pcic_ce_irq_enable(ab, i);
677 	}
678 }
679 EXPORT_SYMBOL(ath11k_pcic_ce_irqs_enable);
680 
681 static void ath11k_pcic_kill_tasklets(struct ath11k_base *ab)
682 {
683 	int i;
684 
685 	for (i = 0; i < ab->hw_params.ce_count; i++) {
686 		struct ath11k_ce_pipe *ce_pipe = &ab->ce.ce_pipe[i];
687 
688 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
689 			continue;
690 
691 		tasklet_kill(&ce_pipe->intr_tq);
692 	}
693 }
694 
695 void ath11k_pcic_ce_irq_disable_sync(struct ath11k_base *ab)
696 {
697 	ath11k_pcic_ce_irqs_disable(ab);
698 	ath11k_pcic_sync_ce_irqs(ab);
699 	ath11k_pcic_kill_tasklets(ab);
700 }
701 EXPORT_SYMBOL(ath11k_pcic_ce_irq_disable_sync);
702 
703 void ath11k_pcic_stop(struct ath11k_base *ab)
704 {
705 	ath11k_pcic_ce_irq_disable_sync(ab);
706 	ath11k_ce_cleanup_pipes(ab);
707 }
708 EXPORT_SYMBOL(ath11k_pcic_stop);
709 
710 int ath11k_pcic_start(struct ath11k_base *ab)
711 {
712 	set_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags);
713 
714 	ath11k_pcic_ce_irqs_enable(ab);
715 	ath11k_ce_rx_post_buf(ab);
716 
717 	return 0;
718 }
719 EXPORT_SYMBOL(ath11k_pcic_start);
720 
721 int ath11k_pcic_map_service_to_pipe(struct ath11k_base *ab, u16 service_id,
722 				    u8 *ul_pipe, u8 *dl_pipe)
723 {
724 	const struct service_to_pipe *entry;
725 	bool ul_set = false, dl_set = false;
726 	int i;
727 
728 	for (i = 0; i < ab->hw_params.svc_to_ce_map_len; i++) {
729 		entry = &ab->hw_params.svc_to_ce_map[i];
730 
731 		if (__le32_to_cpu(entry->service_id) != service_id)
732 			continue;
733 
734 		switch (__le32_to_cpu(entry->pipedir)) {
735 		case PIPEDIR_NONE:
736 			break;
737 		case PIPEDIR_IN:
738 			WARN_ON(dl_set);
739 			*dl_pipe = __le32_to_cpu(entry->pipenum);
740 			dl_set = true;
741 			break;
742 		case PIPEDIR_OUT:
743 			WARN_ON(ul_set);
744 			*ul_pipe = __le32_to_cpu(entry->pipenum);
745 			ul_set = true;
746 			break;
747 		case PIPEDIR_INOUT:
748 			WARN_ON(dl_set);
749 			WARN_ON(ul_set);
750 			*dl_pipe = __le32_to_cpu(entry->pipenum);
751 			*ul_pipe = __le32_to_cpu(entry->pipenum);
752 			dl_set = true;
753 			ul_set = true;
754 			break;
755 		}
756 	}
757 
758 	if (WARN_ON(!ul_set || !dl_set))
759 		return -ENOENT;
760 
761 	return 0;
762 }
763 EXPORT_SYMBOL(ath11k_pcic_map_service_to_pipe);
764 
765 int ath11k_pcic_register_pci_ops(struct ath11k_base *ab,
766 				 const struct ath11k_pci_ops *pci_ops)
767 {
768 	if (!pci_ops)
769 		return 0;
770 
771 	/* Return error if mandatory pci_ops callbacks are missing */
772 	if (!pci_ops->get_msi_irq || !pci_ops->window_write32 ||
773 	    !pci_ops->window_read32)
774 		return -EINVAL;
775 
776 	ab->pci.ops = pci_ops;
777 	return 0;
778 }
779 EXPORT_SYMBOL(ath11k_pcic_register_pci_ops);
780 
781 void ath11k_pci_enable_ce_irqs_except_wake_irq(struct ath11k_base *ab)
782 {
783 	int i;
784 
785 	for (i = 0; i < ab->hw_params.ce_count; i++) {
786 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR ||
787 		    i == ATH11K_PCI_CE_WAKE_IRQ)
788 			continue;
789 		ath11k_pcic_ce_irq_enable(ab, i);
790 	}
791 }
792 EXPORT_SYMBOL(ath11k_pci_enable_ce_irqs_except_wake_irq);
793 
794 void ath11k_pci_disable_ce_irqs_except_wake_irq(struct ath11k_base *ab)
795 {
796 	int i;
797 	int irq_idx;
798 	struct ath11k_ce_pipe *ce_pipe;
799 
800 	for (i = 0; i < ab->hw_params.ce_count; i++) {
801 		ce_pipe = &ab->ce.ce_pipe[i];
802 		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
803 
804 		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR ||
805 		    i == ATH11K_PCI_CE_WAKE_IRQ)
806 			continue;
807 
808 		disable_irq_nosync(ab->irq_num[irq_idx]);
809 		synchronize_irq(ab->irq_num[irq_idx]);
810 		tasklet_kill(&ce_pipe->intr_tq);
811 	}
812 }
813 EXPORT_SYMBOL(ath11k_pci_disable_ce_irqs_except_wake_irq);
814