xref: /openbmc/linux/drivers/net/ipa/ipa_main.c (revision 3d40aed8)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4  * Copyright (C) 2018-2023 Linaro Ltd.
5  */
6 
7 #include <linux/types.h>
8 #include <linux/atomic.h>
9 #include <linux/bitfield.h>
10 #include <linux/device.h>
11 #include <linux/bug.h>
12 #include <linux/io.h>
13 #include <linux/firmware.h>
14 #include <linux/module.h>
15 #include <linux/of.h>
16 #include <linux/of_address.h>
17 #include <linux/platform_device.h>
18 #include <linux/pm_runtime.h>
19 #include <linux/firmware/qcom/qcom_scm.h>
20 #include <linux/soc/qcom/mdt_loader.h>
21 
22 #include "ipa.h"
23 #include "ipa_power.h"
24 #include "ipa_data.h"
25 #include "ipa_endpoint.h"
26 #include "ipa_resource.h"
27 #include "ipa_cmd.h"
28 #include "ipa_reg.h"
29 #include "ipa_mem.h"
30 #include "ipa_table.h"
31 #include "ipa_smp2p.h"
32 #include "ipa_modem.h"
33 #include "ipa_uc.h"
34 #include "ipa_interrupt.h"
35 #include "gsi_trans.h"
36 #include "ipa_sysfs.h"
37 
38 /**
39  * DOC: The IP Accelerator
40  *
41  * This driver supports the Qualcomm IP Accelerator (IPA), which is a
42  * networking component found in many Qualcomm SoCs.  The IPA is connected
43  * to the application processor (AP), but is also connected (and partially
44  * controlled by) other "execution environments" (EEs), such as a modem.
45  *
46  * The IPA is the conduit between the AP and the modem that carries network
47  * traffic.  This driver presents a network interface representing the
48  * connection of the modem to external (e.g. LTE) networks.
49  *
50  * The IPA provides protocol checksum calculation, offloading this work
51  * from the AP.  The IPA offers additional functionality, including routing,
52  * filtering, and NAT support, but that more advanced functionality is not
53  * currently supported.  Despite that, some resources--including routing
54  * tables and filter tables--are defined in this driver because they must
55  * be initialized even when the advanced hardware features are not used.
56  *
57  * There are two distinct layers that implement the IPA hardware, and this
58  * is reflected in the organization of the driver.  The generic software
59  * interface (GSI) is an integral component of the IPA, providing a
60  * well-defined communication layer between the AP subsystem and the IPA
61  * core.  The GSI implements a set of "channels" used for communication
62  * between the AP and the IPA.
63  *
64  * The IPA layer uses GSI channels to implement its "endpoints".  And while
65  * a GSI channel carries data between the AP and the IPA, a pair of IPA
66  * endpoints is used to carry traffic between two EEs.  Specifically, the main
67  * modem network interface is implemented by two pairs of endpoints:  a TX
68  * endpoint on the AP coupled with an RX endpoint on the modem; and another
69  * RX endpoint on the AP receiving data from a TX endpoint on the modem.
70  */
71 
72 /* The name of the GSI firmware file relative to /lib/firmware */
73 #define IPA_FW_PATH_DEFAULT	"ipa_fws.mdt"
74 #define IPA_PAS_ID		15
75 
76 /* Shift of 19.2 MHz timestamp to achieve lower resolution timestamps */
77 #define DPL_TIMESTAMP_SHIFT	14	/* ~1.172 kHz, ~853 usec per tick */
78 #define TAG_TIMESTAMP_SHIFT	14
79 #define NAT_TIMESTAMP_SHIFT	24	/* ~1.144 Hz, ~874 msec per tick */
80 
81 /* Divider for 19.2 MHz crystal oscillator clock to get common timer clock */
82 #define IPA_XO_CLOCK_DIVIDER	192	/* 1 is subtracted where used */
83 
84 /**
85  * enum ipa_firmware_loader: How GSI firmware gets loaded
86  *
87  * @IPA_LOADER_DEFER:		System not ready; try again later
88  * @IPA_LOADER_SELF:		AP loads GSI firmware
89  * @IPA_LOADER_MODEM:		Modem loads GSI firmware, signals when done
90  * @IPA_LOADER_SKIP:		Neither AP nor modem need to load GSI firmware
91  * @IPA_LOADER_INVALID:	GSI firmware loader specification is invalid
92  */
93 enum ipa_firmware_loader {
94 	IPA_LOADER_DEFER,
95 	IPA_LOADER_SELF,
96 	IPA_LOADER_MODEM,
97 	IPA_LOADER_SKIP,
98 	IPA_LOADER_INVALID,
99 };
100 
101 /**
102  * ipa_setup() - Set up IPA hardware
103  * @ipa:	IPA pointer
104  *
105  * Perform initialization that requires issuing immediate commands on
106  * the command TX endpoint.  If the modem is doing GSI firmware load
107  * and initialization, this function will be called when an SMP2P
108  * interrupt has been signaled by the modem.  Otherwise it will be
109  * called from ipa_probe() after GSI firmware has been successfully
110  * loaded, authenticated, and started by Trust Zone.
111  */
ipa_setup(struct ipa * ipa)112 int ipa_setup(struct ipa *ipa)
113 {
114 	struct ipa_endpoint *exception_endpoint;
115 	struct ipa_endpoint *command_endpoint;
116 	struct device *dev = &ipa->pdev->dev;
117 	int ret;
118 
119 	ret = gsi_setup(&ipa->gsi);
120 	if (ret)
121 		return ret;
122 
123 	ret = ipa_power_setup(ipa);
124 	if (ret)
125 		goto err_gsi_teardown;
126 
127 	ipa_endpoint_setup(ipa);
128 
129 	/* We need to use the AP command TX endpoint to perform other
130 	 * initialization, so we enable first.
131 	 */
132 	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];
133 	ret = ipa_endpoint_enable_one(command_endpoint);
134 	if (ret)
135 		goto err_endpoint_teardown;
136 
137 	ret = ipa_mem_setup(ipa);	/* No matching teardown required */
138 	if (ret)
139 		goto err_command_disable;
140 
141 	ret = ipa_table_setup(ipa);	/* No matching teardown required */
142 	if (ret)
143 		goto err_command_disable;
144 
145 	/* Enable the exception handling endpoint, and tell the hardware
146 	 * to use it by default.
147 	 */
148 	exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX];
149 	ret = ipa_endpoint_enable_one(exception_endpoint);
150 	if (ret)
151 		goto err_command_disable;
152 
153 	ipa_endpoint_default_route_set(ipa, exception_endpoint->endpoint_id);
154 
155 	/* We're all set.  Now prepare for communication with the modem */
156 	ret = ipa_qmi_setup(ipa);
157 	if (ret)
158 		goto err_default_route_clear;
159 
160 	ipa->setup_complete = true;
161 
162 	dev_info(dev, "IPA driver setup completed successfully\n");
163 
164 	return 0;
165 
166 err_default_route_clear:
167 	ipa_endpoint_default_route_clear(ipa);
168 	ipa_endpoint_disable_one(exception_endpoint);
169 err_command_disable:
170 	ipa_endpoint_disable_one(command_endpoint);
171 err_endpoint_teardown:
172 	ipa_endpoint_teardown(ipa);
173 	ipa_power_teardown(ipa);
174 err_gsi_teardown:
175 	gsi_teardown(&ipa->gsi);
176 
177 	return ret;
178 }
179 
180 /**
181  * ipa_teardown() - Inverse of ipa_setup()
182  * @ipa:	IPA pointer
183  */
ipa_teardown(struct ipa * ipa)184 static void ipa_teardown(struct ipa *ipa)
185 {
186 	struct ipa_endpoint *exception_endpoint;
187 	struct ipa_endpoint *command_endpoint;
188 
189 	/* We're going to tear everything down, as if setup never completed */
190 	ipa->setup_complete = false;
191 
192 	ipa_qmi_teardown(ipa);
193 	ipa_endpoint_default_route_clear(ipa);
194 	exception_endpoint = ipa->name_map[IPA_ENDPOINT_AP_LAN_RX];
195 	ipa_endpoint_disable_one(exception_endpoint);
196 	command_endpoint = ipa->name_map[IPA_ENDPOINT_AP_COMMAND_TX];
197 	ipa_endpoint_disable_one(command_endpoint);
198 	ipa_endpoint_teardown(ipa);
199 	ipa_power_teardown(ipa);
200 	gsi_teardown(&ipa->gsi);
201 }
202 
203 static void
ipa_hardware_config_bcr(struct ipa * ipa,const struct ipa_data * data)204 ipa_hardware_config_bcr(struct ipa *ipa, const struct ipa_data *data)
205 {
206 	const struct reg *reg;
207 	u32 val;
208 
209 	/* IPA v4.5+ has no backward compatibility register */
210 	if (ipa->version >= IPA_VERSION_4_5)
211 		return;
212 
213 	reg = ipa_reg(ipa, IPA_BCR);
214 	val = data->backward_compat;
215 	iowrite32(val, ipa->reg_virt + reg_offset(reg));
216 }
217 
ipa_hardware_config_tx(struct ipa * ipa)218 static void ipa_hardware_config_tx(struct ipa *ipa)
219 {
220 	enum ipa_version version = ipa->version;
221 	const struct reg *reg;
222 	u32 offset;
223 	u32 val;
224 
225 	if (version <= IPA_VERSION_4_0 || version >= IPA_VERSION_4_5)
226 		return;
227 
228 	/* Disable PA mask to allow HOLB drop */
229 	reg = ipa_reg(ipa, IPA_TX_CFG);
230 	offset = reg_offset(reg);
231 
232 	val = ioread32(ipa->reg_virt + offset);
233 
234 	val &= ~reg_bit(reg, PA_MASK_EN);
235 
236 	iowrite32(val, ipa->reg_virt + offset);
237 }
238 
ipa_hardware_config_clkon(struct ipa * ipa)239 static void ipa_hardware_config_clkon(struct ipa *ipa)
240 {
241 	enum ipa_version version = ipa->version;
242 	const struct reg *reg;
243 	u32 val;
244 
245 	if (version >= IPA_VERSION_4_5)
246 		return;
247 
248 	if (version < IPA_VERSION_4_0 && version != IPA_VERSION_3_1)
249 		return;
250 
251 	/* Implement some hardware workarounds */
252 	reg = ipa_reg(ipa, CLKON_CFG);
253 	if (version == IPA_VERSION_3_1) {
254 		/* Disable MISC clock gating */
255 		val = reg_bit(reg, CLKON_MISC);
256 	} else {	/* IPA v4.0+ */
257 		/* Enable open global clocks in the CLKON configuration */
258 		val = reg_bit(reg, CLKON_GLOBAL);
259 		val |= reg_bit(reg, GLOBAL_2X_CLK);
260 	}
261 
262 	iowrite32(val, ipa->reg_virt + reg_offset(reg));
263 }
264 
265 /* Configure bus access behavior for IPA components */
ipa_hardware_config_comp(struct ipa * ipa)266 static void ipa_hardware_config_comp(struct ipa *ipa)
267 {
268 	const struct reg *reg;
269 	u32 offset;
270 	u32 val;
271 
272 	/* Nothing to configure prior to IPA v4.0 */
273 	if (ipa->version < IPA_VERSION_4_0)
274 		return;
275 
276 	reg = ipa_reg(ipa, COMP_CFG);
277 	offset = reg_offset(reg);
278 
279 	val = ioread32(ipa->reg_virt + offset);
280 
281 	if (ipa->version == IPA_VERSION_4_0) {
282 		val &= ~reg_bit(reg, IPA_QMB_SELECT_CONS_EN);
283 		val &= ~reg_bit(reg, IPA_QMB_SELECT_PROD_EN);
284 		val &= ~reg_bit(reg, IPA_QMB_SELECT_GLOBAL_EN);
285 	} else if (ipa->version < IPA_VERSION_4_5) {
286 		val |= reg_bit(reg, GSI_MULTI_AXI_MASTERS_DIS);
287 	} else {
288 		/* For IPA v4.5+ FULL_FLUSH_WAIT_RS_CLOSURE_EN is 0 */
289 	}
290 
291 	val |= reg_bit(reg, GSI_MULTI_INORDER_RD_DIS);
292 	val |= reg_bit(reg, GSI_MULTI_INORDER_WR_DIS);
293 
294 	iowrite32(val, ipa->reg_virt + offset);
295 }
296 
297 /* Configure DDR and (possibly) PCIe max read/write QSB values */
298 static void
ipa_hardware_config_qsb(struct ipa * ipa,const struct ipa_data * data)299 ipa_hardware_config_qsb(struct ipa *ipa, const struct ipa_data *data)
300 {
301 	const struct ipa_qsb_data *data0;
302 	const struct ipa_qsb_data *data1;
303 	const struct reg *reg;
304 	u32 val;
305 
306 	/* QMB 0 represents DDR; QMB 1 (if present) represents PCIe */
307 	data0 = &data->qsb_data[IPA_QSB_MASTER_DDR];
308 	if (data->qsb_count > 1)
309 		data1 = &data->qsb_data[IPA_QSB_MASTER_PCIE];
310 
311 	/* Max outstanding write accesses for QSB masters */
312 	reg = ipa_reg(ipa, QSB_MAX_WRITES);
313 
314 	val = reg_encode(reg, GEN_QMB_0_MAX_WRITES, data0->max_writes);
315 	if (data->qsb_count > 1)
316 		val |= reg_encode(reg, GEN_QMB_1_MAX_WRITES, data1->max_writes);
317 
318 	iowrite32(val, ipa->reg_virt + reg_offset(reg));
319 
320 	/* Max outstanding read accesses for QSB masters */
321 	reg = ipa_reg(ipa, QSB_MAX_READS);
322 
323 	val = reg_encode(reg, GEN_QMB_0_MAX_READS, data0->max_reads);
324 	if (ipa->version >= IPA_VERSION_4_0)
325 		val |= reg_encode(reg, GEN_QMB_0_MAX_READS_BEATS,
326 				  data0->max_reads_beats);
327 	if (data->qsb_count > 1) {
328 		val = reg_encode(reg, GEN_QMB_1_MAX_READS, data1->max_reads);
329 		if (ipa->version >= IPA_VERSION_4_0)
330 			val |= reg_encode(reg, GEN_QMB_1_MAX_READS_BEATS,
331 					  data1->max_reads_beats);
332 	}
333 
334 	iowrite32(val, ipa->reg_virt + reg_offset(reg));
335 }
336 
337 /* The internal inactivity timer clock is used for the aggregation timer */
338 #define TIMER_FREQUENCY	32000		/* 32 KHz inactivity timer clock */
339 
340 /* Compute the value to use in the COUNTER_CFG register AGGR_GRANULARITY
341  * field to represent the given number of microseconds.  The value is one
342  * less than the number of timer ticks in the requested period.  0 is not
343  * a valid granularity value (so for example @usec must be at least 16 for
344  * a TIMER_FREQUENCY of 32000).
345  */
ipa_aggr_granularity_val(u32 usec)346 static __always_inline u32 ipa_aggr_granularity_val(u32 usec)
347 {
348 	return DIV_ROUND_CLOSEST(usec * TIMER_FREQUENCY, USEC_PER_SEC) - 1;
349 }
350 
351 /* IPA uses unified Qtime starting at IPA v4.5, implementing various
352  * timestamps and timers independent of the IPA core clock rate.  The
353  * Qtimer is based on a 56-bit timestamp incremented at each tick of
354  * a 19.2 MHz SoC crystal oscillator (XO clock).
355  *
356  * For IPA timestamps (tag, NAT, data path logging) a lower resolution
357  * timestamp is achieved by shifting the Qtimer timestamp value right
358  * some number of bits to produce the low-order bits of the coarser
359  * granularity timestamp.
360  *
361  * For timers, a common timer clock is derived from the XO clock using
362  * a divider (we use 192, to produce a 100kHz timer clock).  From
363  * this common clock, three "pulse generators" are used to produce
364  * timer ticks at a configurable frequency.  IPA timers (such as
365  * those used for aggregation or head-of-line block handling) now
366  * define their period based on one of these pulse generators.
367  */
ipa_qtime_config(struct ipa * ipa)368 static void ipa_qtime_config(struct ipa *ipa)
369 {
370 	const struct reg *reg;
371 	u32 offset;
372 	u32 val;
373 
374 	/* Timer clock divider must be disabled when we change the rate */
375 	reg = ipa_reg(ipa, TIMERS_XO_CLK_DIV_CFG);
376 	iowrite32(0, ipa->reg_virt + reg_offset(reg));
377 
378 	reg = ipa_reg(ipa, QTIME_TIMESTAMP_CFG);
379 	/* Set DPL time stamp resolution to use Qtime (instead of 1 msec) */
380 	val = reg_encode(reg, DPL_TIMESTAMP_LSB, DPL_TIMESTAMP_SHIFT);
381 	val |= reg_bit(reg, DPL_TIMESTAMP_SEL);
382 	/* Configure tag and NAT Qtime timestamp resolution as well */
383 	val = reg_encode(reg, TAG_TIMESTAMP_LSB, TAG_TIMESTAMP_SHIFT);
384 	val = reg_encode(reg, NAT_TIMESTAMP_LSB, NAT_TIMESTAMP_SHIFT);
385 
386 	iowrite32(val, ipa->reg_virt + reg_offset(reg));
387 
388 	/* Set granularity of pulse generators used for other timers */
389 	reg = ipa_reg(ipa, TIMERS_PULSE_GRAN_CFG);
390 	val = reg_encode(reg, PULSE_GRAN_0, IPA_GRAN_100_US);
391 	val |= reg_encode(reg, PULSE_GRAN_1, IPA_GRAN_1_MS);
392 	if (ipa->version >= IPA_VERSION_5_0) {
393 		val |= reg_encode(reg, PULSE_GRAN_2, IPA_GRAN_10_MS);
394 		val |= reg_encode(reg, PULSE_GRAN_3, IPA_GRAN_10_MS);
395 	} else {
396 		val |= reg_encode(reg, PULSE_GRAN_2, IPA_GRAN_1_MS);
397 	}
398 
399 	iowrite32(val, ipa->reg_virt + reg_offset(reg));
400 
401 	/* Actual divider is 1 more than value supplied here */
402 	reg = ipa_reg(ipa, TIMERS_XO_CLK_DIV_CFG);
403 	offset = reg_offset(reg);
404 
405 	val = reg_encode(reg, DIV_VALUE, IPA_XO_CLOCK_DIVIDER - 1);
406 
407 	iowrite32(val, ipa->reg_virt + offset);
408 
409 	/* Divider value is set; re-enable the common timer clock divider */
410 	val |= reg_bit(reg, DIV_ENABLE);
411 
412 	iowrite32(val, ipa->reg_virt + offset);
413 }
414 
415 /* Before IPA v4.5 timing is controlled by a counter register */
ipa_hardware_config_counter(struct ipa * ipa)416 static void ipa_hardware_config_counter(struct ipa *ipa)
417 {
418 	u32 granularity = ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY);
419 	const struct reg *reg;
420 	u32 val;
421 
422 	reg = ipa_reg(ipa, COUNTER_CFG);
423 	/* If defined, EOT_COAL_GRANULARITY is 0 */
424 	val = reg_encode(reg, AGGR_GRANULARITY, granularity);
425 	iowrite32(val, ipa->reg_virt + reg_offset(reg));
426 }
427 
ipa_hardware_config_timing(struct ipa * ipa)428 static void ipa_hardware_config_timing(struct ipa *ipa)
429 {
430 	if (ipa->version < IPA_VERSION_4_5)
431 		ipa_hardware_config_counter(ipa);
432 	else
433 		ipa_qtime_config(ipa);
434 }
435 
ipa_hardware_config_hashing(struct ipa * ipa)436 static void ipa_hardware_config_hashing(struct ipa *ipa)
437 {
438 	const struct reg *reg;
439 
440 	/* Other than IPA v4.2, all versions enable "hashing".  Starting
441 	 * with IPA v5.0, the filter and router tables are implemented
442 	 * differently, but the default configuration enables this feature
443 	 * (now referred to as "cacheing"), so there's nothing to do here.
444 	 */
445 	if (ipa->version != IPA_VERSION_4_2)
446 		return;
447 
448 	/* IPA v4.2 does not support hashed tables, so disable them */
449 	reg = ipa_reg(ipa, FILT_ROUT_HASH_EN);
450 
451 	/* IPV6_ROUTER_HASH, IPV6_FILTER_HASH, IPV4_ROUTER_HASH,
452 	 * IPV4_FILTER_HASH are all zero.
453 	 */
454 	iowrite32(0, ipa->reg_virt + reg_offset(reg));
455 }
456 
ipa_idle_indication_cfg(struct ipa * ipa,u32 enter_idle_debounce_thresh,bool const_non_idle_enable)457 static void ipa_idle_indication_cfg(struct ipa *ipa,
458 				    u32 enter_idle_debounce_thresh,
459 				    bool const_non_idle_enable)
460 {
461 	const struct reg *reg;
462 	u32 val;
463 
464 	if (ipa->version < IPA_VERSION_3_5_1)
465 		return;
466 
467 	reg = ipa_reg(ipa, IDLE_INDICATION_CFG);
468 	val = reg_encode(reg, ENTER_IDLE_DEBOUNCE_THRESH,
469 			 enter_idle_debounce_thresh);
470 	if (const_non_idle_enable)
471 		val |= reg_bit(reg, CONST_NON_IDLE_ENABLE);
472 
473 	iowrite32(val, ipa->reg_virt + reg_offset(reg));
474 }
475 
476 /**
477  * ipa_hardware_dcd_config() - Enable dynamic clock division on IPA
478  * @ipa:	IPA pointer
479  *
480  * Configures when the IPA signals it is idle to the global clock
481  * controller, which can respond by scaling down the clock to save
482  * power.
483  */
ipa_hardware_dcd_config(struct ipa * ipa)484 static void ipa_hardware_dcd_config(struct ipa *ipa)
485 {
486 	/* Recommended values for IPA 3.5 and later according to IPA HPG */
487 	ipa_idle_indication_cfg(ipa, 256, false);
488 }
489 
ipa_hardware_dcd_deconfig(struct ipa * ipa)490 static void ipa_hardware_dcd_deconfig(struct ipa *ipa)
491 {
492 	/* Power-on reset values */
493 	ipa_idle_indication_cfg(ipa, 0, true);
494 }
495 
496 /**
497  * ipa_hardware_config() - Primitive hardware initialization
498  * @ipa:	IPA pointer
499  * @data:	IPA configuration data
500  */
ipa_hardware_config(struct ipa * ipa,const struct ipa_data * data)501 static void ipa_hardware_config(struct ipa *ipa, const struct ipa_data *data)
502 {
503 	ipa_hardware_config_bcr(ipa, data);
504 	ipa_hardware_config_tx(ipa);
505 	ipa_hardware_config_clkon(ipa);
506 	ipa_hardware_config_comp(ipa);
507 	ipa_hardware_config_qsb(ipa, data);
508 	ipa_hardware_config_timing(ipa);
509 	ipa_hardware_config_hashing(ipa);
510 	ipa_hardware_dcd_config(ipa);
511 }
512 
513 /**
514  * ipa_hardware_deconfig() - Inverse of ipa_hardware_config()
515  * @ipa:	IPA pointer
516  *
517  * This restores the power-on reset values (even if they aren't different)
518  */
ipa_hardware_deconfig(struct ipa * ipa)519 static void ipa_hardware_deconfig(struct ipa *ipa)
520 {
521 	/* Mostly we just leave things as we set them. */
522 	ipa_hardware_dcd_deconfig(ipa);
523 }
524 
525 /**
526  * ipa_config() - Configure IPA hardware
527  * @ipa:	IPA pointer
528  * @data:	IPA configuration data
529  *
530  * Perform initialization requiring IPA power to be enabled.
531  */
ipa_config(struct ipa * ipa,const struct ipa_data * data)532 static int ipa_config(struct ipa *ipa, const struct ipa_data *data)
533 {
534 	int ret;
535 
536 	ipa_hardware_config(ipa, data);
537 
538 	ret = ipa_mem_config(ipa);
539 	if (ret)
540 		goto err_hardware_deconfig;
541 
542 	ipa->interrupt = ipa_interrupt_config(ipa);
543 	if (IS_ERR(ipa->interrupt)) {
544 		ret = PTR_ERR(ipa->interrupt);
545 		ipa->interrupt = NULL;
546 		goto err_mem_deconfig;
547 	}
548 
549 	ipa_uc_config(ipa);
550 
551 	ret = ipa_endpoint_config(ipa);
552 	if (ret)
553 		goto err_uc_deconfig;
554 
555 	ipa_table_config(ipa);		/* No deconfig required */
556 
557 	/* Assign resource limitation to each group; no deconfig required */
558 	ret = ipa_resource_config(ipa, data->resource_data);
559 	if (ret)
560 		goto err_endpoint_deconfig;
561 
562 	ret = ipa_modem_config(ipa);
563 	if (ret)
564 		goto err_endpoint_deconfig;
565 
566 	return 0;
567 
568 err_endpoint_deconfig:
569 	ipa_endpoint_deconfig(ipa);
570 err_uc_deconfig:
571 	ipa_uc_deconfig(ipa);
572 	ipa_interrupt_deconfig(ipa->interrupt);
573 	ipa->interrupt = NULL;
574 err_mem_deconfig:
575 	ipa_mem_deconfig(ipa);
576 err_hardware_deconfig:
577 	ipa_hardware_deconfig(ipa);
578 
579 	return ret;
580 }
581 
582 /**
583  * ipa_deconfig() - Inverse of ipa_config()
584  * @ipa:	IPA pointer
585  */
ipa_deconfig(struct ipa * ipa)586 static void ipa_deconfig(struct ipa *ipa)
587 {
588 	ipa_modem_deconfig(ipa);
589 	ipa_endpoint_deconfig(ipa);
590 	ipa_uc_deconfig(ipa);
591 	ipa_interrupt_deconfig(ipa->interrupt);
592 	ipa->interrupt = NULL;
593 	ipa_mem_deconfig(ipa);
594 	ipa_hardware_deconfig(ipa);
595 }
596 
ipa_firmware_load(struct device * dev)597 static int ipa_firmware_load(struct device *dev)
598 {
599 	const struct firmware *fw;
600 	struct device_node *node;
601 	struct resource res;
602 	phys_addr_t phys;
603 	const char *path;
604 	ssize_t size;
605 	void *virt;
606 	int ret;
607 
608 	node = of_parse_phandle(dev->of_node, "memory-region", 0);
609 	if (!node) {
610 		dev_err(dev, "DT error getting \"memory-region\" property\n");
611 		return -EINVAL;
612 	}
613 
614 	ret = of_address_to_resource(node, 0, &res);
615 	of_node_put(node);
616 	if (ret) {
617 		dev_err(dev, "error %d getting \"memory-region\" resource\n",
618 			ret);
619 		return ret;
620 	}
621 
622 	/* Use name from DTB if specified; use default for *any* error */
623 	ret = of_property_read_string(dev->of_node, "firmware-name", &path);
624 	if (ret) {
625 		dev_dbg(dev, "error %d getting \"firmware-name\" resource\n",
626 			ret);
627 		path = IPA_FW_PATH_DEFAULT;
628 	}
629 
630 	ret = request_firmware(&fw, path, dev);
631 	if (ret) {
632 		dev_err(dev, "error %d requesting \"%s\"\n", ret, path);
633 		return ret;
634 	}
635 
636 	phys = res.start;
637 	size = (size_t)resource_size(&res);
638 	virt = memremap(phys, size, MEMREMAP_WC);
639 	if (!virt) {
640 		dev_err(dev, "unable to remap firmware memory\n");
641 		ret = -ENOMEM;
642 		goto out_release_firmware;
643 	}
644 
645 	ret = qcom_mdt_load(dev, fw, path, IPA_PAS_ID, virt, phys, size, NULL);
646 	if (ret)
647 		dev_err(dev, "error %d loading \"%s\"\n", ret, path);
648 	else if ((ret = qcom_scm_pas_auth_and_reset(IPA_PAS_ID)))
649 		dev_err(dev, "error %d authenticating \"%s\"\n", ret, path);
650 
651 	memunmap(virt);
652 out_release_firmware:
653 	release_firmware(fw);
654 
655 	return ret;
656 }
657 
658 static const struct of_device_id ipa_match[] = {
659 	{
660 		.compatible	= "qcom,msm8998-ipa",
661 		.data		= &ipa_data_v3_1,
662 	},
663 	{
664 		.compatible	= "qcom,sdm845-ipa",
665 		.data		= &ipa_data_v3_5_1,
666 	},
667 	{
668 		.compatible	= "qcom,sc7180-ipa",
669 		.data		= &ipa_data_v4_2,
670 	},
671 	{
672 		.compatible	= "qcom,sdx55-ipa",
673 		.data		= &ipa_data_v4_5,
674 	},
675 	{
676 		.compatible	= "qcom,sm6350-ipa",
677 		.data		= &ipa_data_v4_7,
678 	},
679 	{
680 		.compatible	= "qcom,sm8350-ipa",
681 		.data		= &ipa_data_v4_9,
682 	},
683 	{
684 		.compatible	= "qcom,sc7280-ipa",
685 		.data		= &ipa_data_v4_11,
686 	},
687 	{
688 		.compatible	= "qcom,sdx65-ipa",
689 		.data		= &ipa_data_v5_0,
690 	},
691 	{ },
692 };
693 MODULE_DEVICE_TABLE(of, ipa_match);
694 
695 /* Check things that can be validated at build time.  This just
696  * groups these things BUILD_BUG_ON() calls don't clutter the rest
697  * of the code.
698  * */
ipa_validate_build(void)699 static void ipa_validate_build(void)
700 {
701 	/* At one time we assumed a 64-bit build, allowing some do_div()
702 	 * calls to be replaced by simple division or modulo operations.
703 	 * We currently only perform divide and modulo operations on u32,
704 	 * u16, or size_t objects, and of those only size_t has any chance
705 	 * of being a 64-bit value.  (It should be guaranteed 32 bits wide
706 	 * on a 32-bit build, but there is no harm in verifying that.)
707 	 */
708 	BUILD_BUG_ON(!IS_ENABLED(CONFIG_64BIT) && sizeof(size_t) != 4);
709 
710 	/* Code assumes the EE ID for the AP is 0 (zeroed structure field) */
711 	BUILD_BUG_ON(GSI_EE_AP != 0);
712 
713 	/* There's no point if we have no channels or event rings */
714 	BUILD_BUG_ON(!GSI_CHANNEL_COUNT_MAX);
715 	BUILD_BUG_ON(!GSI_EVT_RING_COUNT_MAX);
716 
717 	/* GSI hardware design limits */
718 	BUILD_BUG_ON(GSI_CHANNEL_COUNT_MAX > 32);
719 	BUILD_BUG_ON(GSI_EVT_RING_COUNT_MAX > 31);
720 
721 	/* The number of TREs in a transaction is limited by the channel's
722 	 * TLV FIFO size.  A transaction structure uses 8-bit fields
723 	 * to represents the number of TREs it has allocated and used.
724 	 */
725 	BUILD_BUG_ON(GSI_TLV_MAX > U8_MAX);
726 
727 	/* This is used as a divisor */
728 	BUILD_BUG_ON(!IPA_AGGR_GRANULARITY);
729 
730 	/* Aggregation granularity value can't be 0, and must fit */
731 	BUILD_BUG_ON(!ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY));
732 }
733 
ipa_firmware_loader(struct device * dev)734 static enum ipa_firmware_loader ipa_firmware_loader(struct device *dev)
735 {
736 	bool modem_init;
737 	const char *str;
738 	int ret;
739 
740 	/* Look up the old and new properties by name */
741 	modem_init = of_property_read_bool(dev->of_node, "modem-init");
742 	ret = of_property_read_string(dev->of_node, "qcom,gsi-loader", &str);
743 
744 	/* If the new property doesn't exist, it's legacy behavior */
745 	if (ret == -EINVAL) {
746 		if (modem_init)
747 			return IPA_LOADER_MODEM;
748 		goto out_self;
749 	}
750 
751 	/* Any other error on the new property means it's poorly defined */
752 	if (ret)
753 		return IPA_LOADER_INVALID;
754 
755 	/* New property value exists; if old one does too, that's invalid */
756 	if (modem_init)
757 		return IPA_LOADER_INVALID;
758 
759 	/* Modem loads GSI firmware for "modem" */
760 	if (!strcmp(str, "modem"))
761 		return IPA_LOADER_MODEM;
762 
763 	/* No GSI firmware load is needed for "skip" */
764 	if (!strcmp(str, "skip"))
765 		return IPA_LOADER_SKIP;
766 
767 	/* Any value other than "self" is an error */
768 	if (strcmp(str, "self"))
769 		return IPA_LOADER_INVALID;
770 out_self:
771 	/* We need Trust Zone to load firmware; make sure it's available */
772 	if (qcom_scm_is_available())
773 		return IPA_LOADER_SELF;
774 
775 	return IPA_LOADER_DEFER;
776 }
777 
778 /**
779  * ipa_probe() - IPA platform driver probe function
780  * @pdev:	Platform device pointer
781  *
782  * Return:	0 if successful, or a negative error code (possibly
783  *		EPROBE_DEFER)
784  *
785  * This is the main entry point for the IPA driver.  Initialization proceeds
786  * in several stages:
787  *   - The "init" stage involves activities that can be initialized without
788  *     access to the IPA hardware.
789  *   - The "config" stage requires IPA power to be active so IPA registers
790  *     can be accessed, but does not require the use of IPA immediate commands.
791  *   - The "setup" stage uses IPA immediate commands, and so requires the GSI
792  *     layer to be initialized.
793  *
794  * A Boolean Device Tree "modem-init" property determines whether GSI
795  * initialization will be performed by the AP (Trust Zone) or the modem.
796  * If the AP does GSI initialization, the setup phase is entered after
797  * this has completed successfully.  Otherwise the modem initializes
798  * the GSI layer and signals it has finished by sending an SMP2P interrupt
799  * to the AP; this triggers the start if IPA setup.
800  */
ipa_probe(struct platform_device * pdev)801 static int ipa_probe(struct platform_device *pdev)
802 {
803 	struct device *dev = &pdev->dev;
804 	enum ipa_firmware_loader loader;
805 	const struct ipa_data *data;
806 	struct ipa_power *power;
807 	struct ipa *ipa;
808 	int ret;
809 
810 	ipa_validate_build();
811 
812 	/* Get configuration data early; needed for power initialization */
813 	data = of_device_get_match_data(dev);
814 	if (!data) {
815 		dev_err(dev, "matched hardware not supported\n");
816 		return -ENODEV;
817 	}
818 
819 	if (!ipa_version_supported(data->version)) {
820 		dev_err(dev, "unsupported IPA version %u\n", data->version);
821 		return -EINVAL;
822 	}
823 
824 	if (!data->modem_route_count) {
825 		dev_err(dev, "modem_route_count cannot be zero\n");
826 		return -EINVAL;
827 	}
828 
829 	loader = ipa_firmware_loader(dev);
830 	if (loader == IPA_LOADER_INVALID)
831 		return -EINVAL;
832 	if (loader == IPA_LOADER_DEFER)
833 		return -EPROBE_DEFER;
834 
835 	/* The clock and interconnects might not be ready when we're
836 	 * probed, so might return -EPROBE_DEFER.
837 	 */
838 	power = ipa_power_init(dev, data->power_data);
839 	if (IS_ERR(power))
840 		return PTR_ERR(power);
841 
842 	/* No more EPROBE_DEFER.  Allocate and initialize the IPA structure */
843 	ipa = kzalloc(sizeof(*ipa), GFP_KERNEL);
844 	if (!ipa) {
845 		ret = -ENOMEM;
846 		goto err_power_exit;
847 	}
848 
849 	ipa->pdev = pdev;
850 	dev_set_drvdata(dev, ipa);
851 	ipa->power = power;
852 	ipa->version = data->version;
853 	ipa->modem_route_count = data->modem_route_count;
854 	init_completion(&ipa->completion);
855 
856 	ret = ipa_reg_init(ipa);
857 	if (ret)
858 		goto err_kfree_ipa;
859 
860 	ret = ipa_mem_init(ipa, data->mem_data);
861 	if (ret)
862 		goto err_reg_exit;
863 
864 	ret = gsi_init(&ipa->gsi, pdev, ipa->version, data->endpoint_count,
865 		       data->endpoint_data);
866 	if (ret)
867 		goto err_mem_exit;
868 
869 	/* Result is a non-zero mask of endpoints that support filtering */
870 	ret = ipa_endpoint_init(ipa, data->endpoint_count, data->endpoint_data);
871 	if (ret)
872 		goto err_gsi_exit;
873 
874 	ret = ipa_table_init(ipa);
875 	if (ret)
876 		goto err_endpoint_exit;
877 
878 	ret = ipa_smp2p_init(ipa, loader == IPA_LOADER_MODEM);
879 	if (ret)
880 		goto err_table_exit;
881 
882 	/* Power needs to be active for config and setup */
883 	ret = pm_runtime_get_sync(dev);
884 	if (WARN_ON(ret < 0))
885 		goto err_power_put;
886 
887 	ret = ipa_config(ipa, data);
888 	if (ret)
889 		goto err_power_put;
890 
891 	dev_info(dev, "IPA driver initialized");
892 
893 	/* If the modem is loading GSI firmware, it will trigger a call to
894 	 * ipa_setup() when it has finished.  In that case we're done here.
895 	 */
896 	if (loader == IPA_LOADER_MODEM)
897 		goto done;
898 
899 	if (loader == IPA_LOADER_SELF) {
900 		/* The AP is loading GSI firmware; do so now */
901 		ret = ipa_firmware_load(dev);
902 		if (ret)
903 			goto err_deconfig;
904 	} /* Otherwise loader == IPA_LOADER_SKIP */
905 
906 	/* GSI firmware is loaded; proceed to setup */
907 	ret = ipa_setup(ipa);
908 	if (ret)
909 		goto err_deconfig;
910 done:
911 	pm_runtime_mark_last_busy(dev);
912 	(void)pm_runtime_put_autosuspend(dev);
913 
914 	return 0;
915 
916 err_deconfig:
917 	ipa_deconfig(ipa);
918 err_power_put:
919 	pm_runtime_put_noidle(dev);
920 	ipa_smp2p_exit(ipa);
921 err_table_exit:
922 	ipa_table_exit(ipa);
923 err_endpoint_exit:
924 	ipa_endpoint_exit(ipa);
925 err_gsi_exit:
926 	gsi_exit(&ipa->gsi);
927 err_mem_exit:
928 	ipa_mem_exit(ipa);
929 err_reg_exit:
930 	ipa_reg_exit(ipa);
931 err_kfree_ipa:
932 	kfree(ipa);
933 err_power_exit:
934 	ipa_power_exit(power);
935 
936 	return ret;
937 }
938 
ipa_remove(struct platform_device * pdev)939 static int ipa_remove(struct platform_device *pdev)
940 {
941 	struct ipa *ipa = dev_get_drvdata(&pdev->dev);
942 	struct ipa_power *power = ipa->power;
943 	struct device *dev = &pdev->dev;
944 	int ret;
945 
946 	/* Prevent the modem from triggering a call to ipa_setup().  This
947 	 * also ensures a modem-initiated setup that's underway completes.
948 	 */
949 	ipa_smp2p_irq_disable_setup(ipa);
950 
951 	ret = pm_runtime_get_sync(dev);
952 	if (WARN_ON(ret < 0))
953 		goto out_power_put;
954 
955 	if (ipa->setup_complete) {
956 		ret = ipa_modem_stop(ipa);
957 		/* If starting or stopping is in progress, try once more */
958 		if (ret == -EBUSY) {
959 			usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC);
960 			ret = ipa_modem_stop(ipa);
961 		}
962 		if (ret)
963 			return ret;
964 
965 		ipa_teardown(ipa);
966 	}
967 
968 	ipa_deconfig(ipa);
969 out_power_put:
970 	pm_runtime_put_noidle(dev);
971 	ipa_smp2p_exit(ipa);
972 	ipa_table_exit(ipa);
973 	ipa_endpoint_exit(ipa);
974 	gsi_exit(&ipa->gsi);
975 	ipa_mem_exit(ipa);
976 	ipa_reg_exit(ipa);
977 	kfree(ipa);
978 	ipa_power_exit(power);
979 
980 	dev_info(dev, "IPA driver removed");
981 
982 	return 0;
983 }
984 
ipa_shutdown(struct platform_device * pdev)985 static void ipa_shutdown(struct platform_device *pdev)
986 {
987 	int ret;
988 
989 	ret = ipa_remove(pdev);
990 	if (ret)
991 		dev_err(&pdev->dev, "shutdown: remove returned %d\n", ret);
992 }
993 
994 static const struct attribute_group *ipa_attribute_groups[] = {
995 	&ipa_attribute_group,
996 	&ipa_feature_attribute_group,
997 	&ipa_endpoint_id_attribute_group,
998 	&ipa_modem_attribute_group,
999 	NULL,
1000 };
1001 
1002 static struct platform_driver ipa_driver = {
1003 	.probe		= ipa_probe,
1004 	.remove		= ipa_remove,
1005 	.shutdown	= ipa_shutdown,
1006 	.driver	= {
1007 		.name		= "ipa",
1008 		.pm		= &ipa_pm_ops,
1009 		.of_match_table	= ipa_match,
1010 		.dev_groups	= ipa_attribute_groups,
1011 	},
1012 };
1013 
1014 module_platform_driver(ipa_driver);
1015 
1016 MODULE_LICENSE("GPL v2");
1017 MODULE_DESCRIPTION("Qualcomm IP Accelerator device driver");
1018