1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2013-2016, Linux Foundation. All rights reserved.
4 */
5
6 #include <linux/acpi.h>
7 #include <linux/time.h>
8 #include <linux/clk.h>
9 #include <linux/delay.h>
10 #include <linux/interconnect.h>
11 #include <linux/module.h>
12 #include <linux/of.h>
13 #include <linux/platform_device.h>
14 #include <linux/phy/phy.h>
15 #include <linux/gpio/consumer.h>
16 #include <linux/reset-controller.h>
17 #include <linux/devfreq.h>
18
19 #include <soc/qcom/ice.h>
20
21 #include <ufs/ufshcd.h>
22 #include "ufshcd-pltfrm.h"
23 #include <ufs/unipro.h>
24 #include "ufs-qcom.h"
25 #include <ufs/ufshci.h>
26 #include <ufs/ufs_quirks.h>
27
28 #define MCQ_QCFGPTR_MASK GENMASK(7, 0)
29 #define MCQ_QCFGPTR_UNIT 0x200
30 #define MCQ_SQATTR_OFFSET(c) \
31 ((((c) >> 16) & MCQ_QCFGPTR_MASK) * MCQ_QCFGPTR_UNIT)
32 #define MCQ_QCFG_SIZE 0x40
33
34 enum {
35 TSTBUS_UAWM,
36 TSTBUS_UARM,
37 TSTBUS_TXUC,
38 TSTBUS_RXUC,
39 TSTBUS_DFC,
40 TSTBUS_TRLUT,
41 TSTBUS_TMRLUT,
42 TSTBUS_OCSC,
43 TSTBUS_UTP_HCI,
44 TSTBUS_COMBINED,
45 TSTBUS_WRAPPER,
46 TSTBUS_UNIPRO,
47 TSTBUS_MAX,
48 };
49
50 #define QCOM_UFS_MAX_GEAR 5
51 #define QCOM_UFS_MAX_LANE 2
52
53 enum {
54 MODE_MIN,
55 MODE_PWM,
56 MODE_HS_RA,
57 MODE_HS_RB,
58 MODE_MAX,
59 };
60
61 static const struct __ufs_qcom_bw_table {
62 u32 mem_bw;
63 u32 cfg_bw;
64 } ufs_qcom_bw_table[MODE_MAX + 1][QCOM_UFS_MAX_GEAR + 1][QCOM_UFS_MAX_LANE + 1] = {
65 [MODE_MIN][0][0] = { 0, 0 }, /* Bandwidth values in KB/s */
66 [MODE_PWM][UFS_PWM_G1][UFS_LANE_1] = { 922, 1000 },
67 [MODE_PWM][UFS_PWM_G2][UFS_LANE_1] = { 1844, 1000 },
68 [MODE_PWM][UFS_PWM_G3][UFS_LANE_1] = { 3688, 1000 },
69 [MODE_PWM][UFS_PWM_G4][UFS_LANE_1] = { 7376, 1000 },
70 [MODE_PWM][UFS_PWM_G5][UFS_LANE_1] = { 14752, 1000 },
71 [MODE_PWM][UFS_PWM_G1][UFS_LANE_2] = { 1844, 1000 },
72 [MODE_PWM][UFS_PWM_G2][UFS_LANE_2] = { 3688, 1000 },
73 [MODE_PWM][UFS_PWM_G3][UFS_LANE_2] = { 7376, 1000 },
74 [MODE_PWM][UFS_PWM_G4][UFS_LANE_2] = { 14752, 1000 },
75 [MODE_PWM][UFS_PWM_G5][UFS_LANE_2] = { 29504, 1000 },
76 [MODE_HS_RA][UFS_HS_G1][UFS_LANE_1] = { 127796, 1000 },
77 [MODE_HS_RA][UFS_HS_G2][UFS_LANE_1] = { 255591, 1000 },
78 [MODE_HS_RA][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 },
79 [MODE_HS_RA][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 },
80 [MODE_HS_RA][UFS_HS_G5][UFS_LANE_1] = { 5836800, 409600 },
81 [MODE_HS_RA][UFS_HS_G1][UFS_LANE_2] = { 255591, 1000 },
82 [MODE_HS_RA][UFS_HS_G2][UFS_LANE_2] = { 511181, 1000 },
83 [MODE_HS_RA][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 },
84 [MODE_HS_RA][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 },
85 [MODE_HS_RA][UFS_HS_G5][UFS_LANE_2] = { 5836800, 819200 },
86 [MODE_HS_RB][UFS_HS_G1][UFS_LANE_1] = { 149422, 1000 },
87 [MODE_HS_RB][UFS_HS_G2][UFS_LANE_1] = { 298189, 1000 },
88 [MODE_HS_RB][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 },
89 [MODE_HS_RB][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 },
90 [MODE_HS_RB][UFS_HS_G5][UFS_LANE_1] = { 5836800, 409600 },
91 [MODE_HS_RB][UFS_HS_G1][UFS_LANE_2] = { 298189, 1000 },
92 [MODE_HS_RB][UFS_HS_G2][UFS_LANE_2] = { 596378, 1000 },
93 [MODE_HS_RB][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 },
94 [MODE_HS_RB][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 },
95 [MODE_HS_RB][UFS_HS_G5][UFS_LANE_2] = { 5836800, 819200 },
96 [MODE_MAX][0][0] = { 7643136, 819200 },
97 };
98
99 static struct ufs_qcom_host *ufs_qcom_hosts[MAX_UFS_QCOM_HOSTS];
100
101 static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host);
102 static int ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba *hba,
103 u32 clk_cycles);
104
rcdev_to_ufs_host(struct reset_controller_dev * rcd)105 static struct ufs_qcom_host *rcdev_to_ufs_host(struct reset_controller_dev *rcd)
106 {
107 return container_of(rcd, struct ufs_qcom_host, rcdev);
108 }
109
110 #ifdef CONFIG_SCSI_UFS_CRYPTO
111
ufs_qcom_ice_enable(struct ufs_qcom_host * host)112 static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host)
113 {
114 if (host->hba->caps & UFSHCD_CAP_CRYPTO)
115 qcom_ice_enable(host->ice);
116 }
117
ufs_qcom_ice_init(struct ufs_qcom_host * host)118 static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
119 {
120 struct ufs_hba *hba = host->hba;
121 struct device *dev = hba->dev;
122 struct qcom_ice *ice;
123
124 ice = devm_of_qcom_ice_get(dev);
125 if (ice == ERR_PTR(-EOPNOTSUPP)) {
126 dev_warn(dev, "Disabling inline encryption support\n");
127 ice = NULL;
128 }
129
130 if (IS_ERR_OR_NULL(ice))
131 return PTR_ERR_OR_ZERO(ice);
132
133 host->ice = ice;
134 hba->caps |= UFSHCD_CAP_CRYPTO;
135
136 return 0;
137 }
138
ufs_qcom_ice_resume(struct ufs_qcom_host * host)139 static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
140 {
141 if (host->hba->caps & UFSHCD_CAP_CRYPTO)
142 return qcom_ice_resume(host->ice);
143
144 return 0;
145 }
146
ufs_qcom_ice_suspend(struct ufs_qcom_host * host)147 static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host)
148 {
149 if (host->hba->caps & UFSHCD_CAP_CRYPTO)
150 return qcom_ice_suspend(host->ice);
151
152 return 0;
153 }
154
ufs_qcom_ice_program_key(struct ufs_hba * hba,const union ufs_crypto_cfg_entry * cfg,int slot)155 static int ufs_qcom_ice_program_key(struct ufs_hba *hba,
156 const union ufs_crypto_cfg_entry *cfg,
157 int slot)
158 {
159 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
160 union ufs_crypto_cap_entry cap;
161
162 if (!(cfg->config_enable & UFS_CRYPTO_CONFIGURATION_ENABLE))
163 return qcom_ice_evict_key(host->ice, slot);
164
165 /* Only AES-256-XTS has been tested so far. */
166 cap = hba->crypto_cap_array[cfg->crypto_cap_idx];
167 if (cap.algorithm_id != UFS_CRYPTO_ALG_AES_XTS ||
168 cap.key_size != UFS_CRYPTO_KEY_SIZE_256)
169 return -EOPNOTSUPP;
170
171 return qcom_ice_program_key(host->ice,
172 QCOM_ICE_CRYPTO_ALG_AES_XTS,
173 QCOM_ICE_CRYPTO_KEY_SIZE_256,
174 cfg->crypto_key,
175 cfg->data_unit_size, slot);
176 }
177
178 #else
179
180 #define ufs_qcom_ice_program_key NULL
181
ufs_qcom_ice_enable(struct ufs_qcom_host * host)182 static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host)
183 {
184 }
185
ufs_qcom_ice_init(struct ufs_qcom_host * host)186 static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
187 {
188 return 0;
189 }
190
ufs_qcom_ice_resume(struct ufs_qcom_host * host)191 static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host)
192 {
193 return 0;
194 }
195
ufs_qcom_ice_suspend(struct ufs_qcom_host * host)196 static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host)
197 {
198 return 0;
199 }
200 #endif
201
ufs_qcom_host_clk_get(struct device * dev,const char * name,struct clk ** clk_out,bool optional)202 static int ufs_qcom_host_clk_get(struct device *dev,
203 const char *name, struct clk **clk_out, bool optional)
204 {
205 struct clk *clk;
206 int err = 0;
207
208 clk = devm_clk_get(dev, name);
209 if (!IS_ERR(clk)) {
210 *clk_out = clk;
211 return 0;
212 }
213
214 err = PTR_ERR(clk);
215
216 if (optional && err == -ENOENT) {
217 *clk_out = NULL;
218 return 0;
219 }
220
221 if (err != -EPROBE_DEFER)
222 dev_err(dev, "failed to get %s err %d\n", name, err);
223
224 return err;
225 }
226
ufs_qcom_host_clk_enable(struct device * dev,const char * name,struct clk * clk)227 static int ufs_qcom_host_clk_enable(struct device *dev,
228 const char *name, struct clk *clk)
229 {
230 int err = 0;
231
232 err = clk_prepare_enable(clk);
233 if (err)
234 dev_err(dev, "%s: %s enable failed %d\n", __func__, name, err);
235
236 return err;
237 }
238
ufs_qcom_disable_lane_clks(struct ufs_qcom_host * host)239 static void ufs_qcom_disable_lane_clks(struct ufs_qcom_host *host)
240 {
241 if (!host->is_lane_clks_enabled)
242 return;
243
244 clk_disable_unprepare(host->tx_l1_sync_clk);
245 clk_disable_unprepare(host->tx_l0_sync_clk);
246 clk_disable_unprepare(host->rx_l1_sync_clk);
247 clk_disable_unprepare(host->rx_l0_sync_clk);
248
249 host->is_lane_clks_enabled = false;
250 }
251
ufs_qcom_enable_lane_clks(struct ufs_qcom_host * host)252 static int ufs_qcom_enable_lane_clks(struct ufs_qcom_host *host)
253 {
254 int err;
255 struct device *dev = host->hba->dev;
256
257 if (host->is_lane_clks_enabled)
258 return 0;
259
260 err = ufs_qcom_host_clk_enable(dev, "rx_lane0_sync_clk",
261 host->rx_l0_sync_clk);
262 if (err)
263 return err;
264
265 err = ufs_qcom_host_clk_enable(dev, "tx_lane0_sync_clk",
266 host->tx_l0_sync_clk);
267 if (err)
268 goto disable_rx_l0;
269
270 err = ufs_qcom_host_clk_enable(dev, "rx_lane1_sync_clk",
271 host->rx_l1_sync_clk);
272 if (err)
273 goto disable_tx_l0;
274
275 err = ufs_qcom_host_clk_enable(dev, "tx_lane1_sync_clk",
276 host->tx_l1_sync_clk);
277 if (err)
278 goto disable_rx_l1;
279
280 host->is_lane_clks_enabled = true;
281
282 return 0;
283
284 disable_rx_l1:
285 clk_disable_unprepare(host->rx_l1_sync_clk);
286 disable_tx_l0:
287 clk_disable_unprepare(host->tx_l0_sync_clk);
288 disable_rx_l0:
289 clk_disable_unprepare(host->rx_l0_sync_clk);
290
291 return err;
292 }
293
ufs_qcom_init_lane_clks(struct ufs_qcom_host * host)294 static int ufs_qcom_init_lane_clks(struct ufs_qcom_host *host)
295 {
296 int err = 0;
297 struct device *dev = host->hba->dev;
298
299 if (has_acpi_companion(dev))
300 return 0;
301
302 err = ufs_qcom_host_clk_get(dev, "rx_lane0_sync_clk",
303 &host->rx_l0_sync_clk, false);
304 if (err)
305 return err;
306
307 err = ufs_qcom_host_clk_get(dev, "tx_lane0_sync_clk",
308 &host->tx_l0_sync_clk, false);
309 if (err)
310 return err;
311
312 /* In case of single lane per direction, don't read lane1 clocks */
313 if (host->hba->lanes_per_direction > 1) {
314 err = ufs_qcom_host_clk_get(dev, "rx_lane1_sync_clk",
315 &host->rx_l1_sync_clk, false);
316 if (err)
317 return err;
318
319 err = ufs_qcom_host_clk_get(dev, "tx_lane1_sync_clk",
320 &host->tx_l1_sync_clk, true);
321 }
322
323 return 0;
324 }
325
ufs_qcom_check_hibern8(struct ufs_hba * hba)326 static int ufs_qcom_check_hibern8(struct ufs_hba *hba)
327 {
328 int err;
329 u32 tx_fsm_val = 0;
330 unsigned long timeout = jiffies + msecs_to_jiffies(HBRN8_POLL_TOUT_MS);
331
332 do {
333 err = ufshcd_dme_get(hba,
334 UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE,
335 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
336 &tx_fsm_val);
337 if (err || tx_fsm_val == TX_FSM_HIBERN8)
338 break;
339
340 /* sleep for max. 200us */
341 usleep_range(100, 200);
342 } while (time_before(jiffies, timeout));
343
344 /*
345 * we might have scheduled out for long during polling so
346 * check the state again.
347 */
348 if (time_after(jiffies, timeout))
349 err = ufshcd_dme_get(hba,
350 UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE,
351 UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
352 &tx_fsm_val);
353
354 if (err) {
355 dev_err(hba->dev, "%s: unable to get TX_FSM_STATE, err %d\n",
356 __func__, err);
357 } else if (tx_fsm_val != TX_FSM_HIBERN8) {
358 err = tx_fsm_val;
359 dev_err(hba->dev, "%s: invalid TX_FSM_STATE = %d\n",
360 __func__, err);
361 }
362
363 return err;
364 }
365
ufs_qcom_select_unipro_mode(struct ufs_qcom_host * host)366 static void ufs_qcom_select_unipro_mode(struct ufs_qcom_host *host)
367 {
368 ufshcd_rmwl(host->hba, QUNIPRO_SEL,
369 ufs_qcom_cap_qunipro(host) ? QUNIPRO_SEL : 0,
370 REG_UFS_CFG1);
371
372 if (host->hw_ver.major >= 0x05)
373 ufshcd_rmwl(host->hba, QUNIPRO_G4_SEL, 0, REG_UFS_CFG0);
374 }
375
376 /*
377 * ufs_qcom_host_reset - reset host controller and PHY
378 */
ufs_qcom_host_reset(struct ufs_hba * hba)379 static int ufs_qcom_host_reset(struct ufs_hba *hba)
380 {
381 int ret = 0;
382 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
383 bool reenable_intr = false;
384
385 if (!host->core_reset) {
386 dev_warn(hba->dev, "%s: reset control not set\n", __func__);
387 return 0;
388 }
389
390 reenable_intr = hba->is_irq_enabled;
391 disable_irq(hba->irq);
392 hba->is_irq_enabled = false;
393
394 ret = reset_control_assert(host->core_reset);
395 if (ret) {
396 dev_err(hba->dev, "%s: core_reset assert failed, err = %d\n",
397 __func__, ret);
398 return ret;
399 }
400
401 /*
402 * The hardware requirement for delay between assert/deassert
403 * is at least 3-4 sleep clock (32.7KHz) cycles, which comes to
404 * ~125us (4/32768). To be on the safe side add 200us delay.
405 */
406 usleep_range(200, 210);
407
408 ret = reset_control_deassert(host->core_reset);
409 if (ret)
410 dev_err(hba->dev, "%s: core_reset deassert failed, err = %d\n",
411 __func__, ret);
412
413 usleep_range(1000, 1100);
414
415 if (reenable_intr) {
416 enable_irq(hba->irq);
417 hba->is_irq_enabled = true;
418 }
419
420 return 0;
421 }
422
ufs_qcom_get_hs_gear(struct ufs_hba * hba)423 static u32 ufs_qcom_get_hs_gear(struct ufs_hba *hba)
424 {
425 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
426
427 if (host->hw_ver.major == 0x1) {
428 /*
429 * HS-G3 operations may not reliably work on legacy QCOM
430 * UFS host controller hardware even though capability
431 * exchange during link startup phase may end up
432 * negotiating maximum supported gear as G3.
433 * Hence downgrade the maximum supported gear to HS-G2.
434 */
435 return UFS_HS_G2;
436 } else if (host->hw_ver.major >= 0x4) {
437 return UFS_QCOM_MAX_GEAR(ufshcd_readl(hba, REG_UFS_PARAM0));
438 }
439
440 /* Default is HS-G3 */
441 return UFS_HS_G3;
442 }
443
ufs_qcom_power_up_sequence(struct ufs_hba * hba)444 static int ufs_qcom_power_up_sequence(struct ufs_hba *hba)
445 {
446 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
447 struct phy *phy = host->generic_phy;
448 int ret;
449
450 /* Reset UFS Host Controller and PHY */
451 ret = ufs_qcom_host_reset(hba);
452 if (ret)
453 dev_warn(hba->dev, "%s: host reset returned %d\n",
454 __func__, ret);
455
456 if (phy->power_count)
457 phy_power_off(phy);
458
459
460 /* phy initialization - calibrate the phy */
461 ret = phy_init(phy);
462 if (ret) {
463 dev_err(hba->dev, "%s: phy init failed, ret = %d\n",
464 __func__, ret);
465 return ret;
466 }
467
468 phy_set_mode_ext(phy, PHY_MODE_UFS_HS_B, host->hs_gear);
469
470 /* power on phy - start serdes and phy's power and clocks */
471 ret = phy_power_on(phy);
472 if (ret) {
473 dev_err(hba->dev, "%s: phy power on failed, ret = %d\n",
474 __func__, ret);
475 goto out_disable_phy;
476 }
477
478 ufs_qcom_select_unipro_mode(host);
479
480 return 0;
481
482 out_disable_phy:
483 phy_exit(phy);
484
485 return ret;
486 }
487
488 /*
489 * The UTP controller has a number of internal clock gating cells (CGCs).
490 * Internal hardware sub-modules within the UTP controller control the CGCs.
491 * Hardware CGCs disable the clock to inactivate UTP sub-modules not involved
492 * in a specific operation, UTP controller CGCs are by default disabled and
493 * this function enables them (after every UFS link startup) to save some power
494 * leakage.
495 */
ufs_qcom_enable_hw_clk_gating(struct ufs_hba * hba)496 static void ufs_qcom_enable_hw_clk_gating(struct ufs_hba *hba)
497 {
498 ufshcd_writel(hba,
499 ufshcd_readl(hba, REG_UFS_CFG2) | REG_UFS_CFG2_CGC_EN_ALL,
500 REG_UFS_CFG2);
501
502 /* Ensure that HW clock gating is enabled before next operations */
503 ufshcd_readl(hba, REG_UFS_CFG2);
504 }
505
ufs_qcom_hce_enable_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)506 static int ufs_qcom_hce_enable_notify(struct ufs_hba *hba,
507 enum ufs_notify_change_status status)
508 {
509 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
510 int err = 0;
511
512 switch (status) {
513 case PRE_CHANGE:
514 ufs_qcom_power_up_sequence(hba);
515 /*
516 * The PHY PLL output is the source of tx/rx lane symbol
517 * clocks, hence, enable the lane clocks only after PHY
518 * is initialized.
519 */
520 err = ufs_qcom_enable_lane_clks(host);
521 break;
522 case POST_CHANGE:
523 /* check if UFS PHY moved from DISABLED to HIBERN8 */
524 err = ufs_qcom_check_hibern8(hba);
525 ufs_qcom_enable_hw_clk_gating(hba);
526 ufs_qcom_ice_enable(host);
527 break;
528 default:
529 dev_err(hba->dev, "%s: invalid status %d\n", __func__, status);
530 err = -EINVAL;
531 break;
532 }
533 return err;
534 }
535
536 /*
537 * Return: zero for success and non-zero in case of a failure.
538 */
ufs_qcom_cfg_timers(struct ufs_hba * hba,u32 gear,u32 hs,u32 rate,bool update_link_startup_timer)539 static int ufs_qcom_cfg_timers(struct ufs_hba *hba, u32 gear,
540 u32 hs, u32 rate, bool update_link_startup_timer)
541 {
542 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
543 struct ufs_clk_info *clki;
544 u32 core_clk_period_in_ns;
545 u32 tx_clk_cycles_per_us = 0;
546 unsigned long core_clk_rate = 0;
547 u32 core_clk_cycles_per_us = 0;
548
549 static u32 pwm_fr_table[][2] = {
550 {UFS_PWM_G1, 0x1},
551 {UFS_PWM_G2, 0x1},
552 {UFS_PWM_G3, 0x1},
553 {UFS_PWM_G4, 0x1},
554 };
555
556 static u32 hs_fr_table_rA[][2] = {
557 {UFS_HS_G1, 0x1F},
558 {UFS_HS_G2, 0x3e},
559 {UFS_HS_G3, 0x7D},
560 };
561
562 static u32 hs_fr_table_rB[][2] = {
563 {UFS_HS_G1, 0x24},
564 {UFS_HS_G2, 0x49},
565 {UFS_HS_G3, 0x92},
566 };
567
568 /*
569 * The Qunipro controller does not use following registers:
570 * SYS1CLK_1US_REG, TX_SYMBOL_CLK_1US_REG, CLK_NS_REG &
571 * UFS_REG_PA_LINK_STARTUP_TIMER
572 * But UTP controller uses SYS1CLK_1US_REG register for Interrupt
573 * Aggregation logic.
574 */
575 if (ufs_qcom_cap_qunipro(host) && !ufshcd_is_intr_aggr_allowed(hba))
576 return 0;
577
578 if (gear == 0) {
579 dev_err(hba->dev, "%s: invalid gear = %d\n", __func__, gear);
580 return -EINVAL;
581 }
582
583 list_for_each_entry(clki, &hba->clk_list_head, list) {
584 if (!strcmp(clki->name, "core_clk"))
585 core_clk_rate = clk_get_rate(clki->clk);
586 }
587
588 /* If frequency is smaller than 1MHz, set to 1MHz */
589 if (core_clk_rate < DEFAULT_CLK_RATE_HZ)
590 core_clk_rate = DEFAULT_CLK_RATE_HZ;
591
592 core_clk_cycles_per_us = core_clk_rate / USEC_PER_SEC;
593 if (ufshcd_readl(hba, REG_UFS_SYS1CLK_1US) != core_clk_cycles_per_us) {
594 ufshcd_writel(hba, core_clk_cycles_per_us, REG_UFS_SYS1CLK_1US);
595 /*
596 * make sure above write gets applied before we return from
597 * this function.
598 */
599 ufshcd_readl(hba, REG_UFS_SYS1CLK_1US);
600 }
601
602 if (ufs_qcom_cap_qunipro(host))
603 return 0;
604
605 core_clk_period_in_ns = NSEC_PER_SEC / core_clk_rate;
606 core_clk_period_in_ns <<= OFFSET_CLK_NS_REG;
607 core_clk_period_in_ns &= MASK_CLK_NS_REG;
608
609 switch (hs) {
610 case FASTAUTO_MODE:
611 case FAST_MODE:
612 if (rate == PA_HS_MODE_A) {
613 if (gear > ARRAY_SIZE(hs_fr_table_rA)) {
614 dev_err(hba->dev,
615 "%s: index %d exceeds table size %zu\n",
616 __func__, gear,
617 ARRAY_SIZE(hs_fr_table_rA));
618 return -EINVAL;
619 }
620 tx_clk_cycles_per_us = hs_fr_table_rA[gear-1][1];
621 } else if (rate == PA_HS_MODE_B) {
622 if (gear > ARRAY_SIZE(hs_fr_table_rB)) {
623 dev_err(hba->dev,
624 "%s: index %d exceeds table size %zu\n",
625 __func__, gear,
626 ARRAY_SIZE(hs_fr_table_rB));
627 return -EINVAL;
628 }
629 tx_clk_cycles_per_us = hs_fr_table_rB[gear-1][1];
630 } else {
631 dev_err(hba->dev, "%s: invalid rate = %d\n",
632 __func__, rate);
633 return -EINVAL;
634 }
635 break;
636 case SLOWAUTO_MODE:
637 case SLOW_MODE:
638 if (gear > ARRAY_SIZE(pwm_fr_table)) {
639 dev_err(hba->dev,
640 "%s: index %d exceeds table size %zu\n",
641 __func__, gear,
642 ARRAY_SIZE(pwm_fr_table));
643 return -EINVAL;
644 }
645 tx_clk_cycles_per_us = pwm_fr_table[gear-1][1];
646 break;
647 case UNCHANGED:
648 default:
649 dev_err(hba->dev, "%s: invalid mode = %d\n", __func__, hs);
650 return -EINVAL;
651 }
652
653 if (ufshcd_readl(hba, REG_UFS_TX_SYMBOL_CLK_NS_US) !=
654 (core_clk_period_in_ns | tx_clk_cycles_per_us)) {
655 /* this register 2 fields shall be written at once */
656 ufshcd_writel(hba, core_clk_period_in_ns | tx_clk_cycles_per_us,
657 REG_UFS_TX_SYMBOL_CLK_NS_US);
658 /*
659 * make sure above write gets applied before we return from
660 * this function.
661 */
662 mb();
663 }
664
665 if (update_link_startup_timer && host->hw_ver.major != 0x5) {
666 ufshcd_writel(hba, ((core_clk_rate / MSEC_PER_SEC) * 100),
667 REG_UFS_CFG0);
668 /*
669 * make sure that this configuration is applied before
670 * we return
671 */
672 mb();
673 }
674
675 return 0;
676 }
677
ufs_qcom_link_startup_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)678 static int ufs_qcom_link_startup_notify(struct ufs_hba *hba,
679 enum ufs_notify_change_status status)
680 {
681 int err = 0;
682 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
683
684 switch (status) {
685 case PRE_CHANGE:
686 if (ufs_qcom_cfg_timers(hba, UFS_PWM_G1, SLOWAUTO_MODE,
687 0, true)) {
688 dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n",
689 __func__);
690 return -EINVAL;
691 }
692
693 if (ufs_qcom_cap_qunipro(host))
694 /*
695 * set unipro core clock cycles to 150 & clear clock
696 * divider
697 */
698 err = ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba,
699 150);
700
701 /*
702 * Some UFS devices (and may be host) have issues if LCC is
703 * enabled. So we are setting PA_Local_TX_LCC_Enable to 0
704 * before link startup which will make sure that both host
705 * and device TX LCC are disabled once link startup is
706 * completed.
707 */
708 if (ufshcd_get_local_unipro_ver(hba) != UFS_UNIPRO_VER_1_41)
709 err = ufshcd_disable_host_tx_lcc(hba);
710
711 break;
712 default:
713 break;
714 }
715
716 return err;
717 }
718
ufs_qcom_device_reset_ctrl(struct ufs_hba * hba,bool asserted)719 static void ufs_qcom_device_reset_ctrl(struct ufs_hba *hba, bool asserted)
720 {
721 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
722
723 /* reset gpio is optional */
724 if (!host->device_reset)
725 return;
726
727 gpiod_set_value_cansleep(host->device_reset, asserted);
728 }
729
ufs_qcom_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op,enum ufs_notify_change_status status)730 static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
731 enum ufs_notify_change_status status)
732 {
733 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
734 struct phy *phy = host->generic_phy;
735
736 if (status == PRE_CHANGE)
737 return 0;
738
739 if (ufs_qcom_is_link_off(hba)) {
740 /*
741 * Disable the tx/rx lane symbol clocks before PHY is
742 * powered down as the PLL source should be disabled
743 * after downstream clocks are disabled.
744 */
745 ufs_qcom_disable_lane_clks(host);
746 phy_power_off(phy);
747
748 /* reset the connected UFS device during power down */
749 ufs_qcom_device_reset_ctrl(hba, true);
750
751 } else if (!ufs_qcom_is_link_active(hba)) {
752 ufs_qcom_disable_lane_clks(host);
753 }
754
755 return ufs_qcom_ice_suspend(host);
756 }
757
ufs_qcom_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)758 static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
759 {
760 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
761 struct phy *phy = host->generic_phy;
762 int err;
763
764 if (ufs_qcom_is_link_off(hba)) {
765 err = phy_power_on(phy);
766 if (err) {
767 dev_err(hba->dev, "%s: failed PHY power on: %d\n",
768 __func__, err);
769 return err;
770 }
771
772 err = ufs_qcom_enable_lane_clks(host);
773 if (err)
774 return err;
775
776 } else if (!ufs_qcom_is_link_active(hba)) {
777 err = ufs_qcom_enable_lane_clks(host);
778 if (err)
779 return err;
780 }
781
782 return ufs_qcom_ice_resume(host);
783 }
784
ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host * host,bool enable)785 static void ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host *host, bool enable)
786 {
787 if (host->dev_ref_clk_ctrl_mmio &&
788 (enable ^ host->is_dev_ref_clk_enabled)) {
789 u32 temp = readl_relaxed(host->dev_ref_clk_ctrl_mmio);
790
791 if (enable)
792 temp |= host->dev_ref_clk_en_mask;
793 else
794 temp &= ~host->dev_ref_clk_en_mask;
795
796 /*
797 * If we are here to disable this clock it might be immediately
798 * after entering into hibern8 in which case we need to make
799 * sure that device ref_clk is active for specific time after
800 * hibern8 enter.
801 */
802 if (!enable) {
803 unsigned long gating_wait;
804
805 gating_wait = host->hba->dev_info.clk_gating_wait_us;
806 if (!gating_wait) {
807 udelay(1);
808 } else {
809 /*
810 * bRefClkGatingWaitTime defines the minimum
811 * time for which the reference clock is
812 * required by device during transition from
813 * HS-MODE to LS-MODE or HIBERN8 state. Give it
814 * more delay to be on the safe side.
815 */
816 gating_wait += 10;
817 usleep_range(gating_wait, gating_wait + 10);
818 }
819 }
820
821 writel_relaxed(temp, host->dev_ref_clk_ctrl_mmio);
822
823 /*
824 * Make sure the write to ref_clk reaches the destination and
825 * not stored in a Write Buffer (WB).
826 */
827 readl(host->dev_ref_clk_ctrl_mmio);
828
829 /*
830 * If we call hibern8 exit after this, we need to make sure that
831 * device ref_clk is stable for at least 1us before the hibern8
832 * exit command.
833 */
834 if (enable)
835 udelay(1);
836
837 host->is_dev_ref_clk_enabled = enable;
838 }
839 }
840
ufs_qcom_icc_set_bw(struct ufs_qcom_host * host,u32 mem_bw,u32 cfg_bw)841 static int ufs_qcom_icc_set_bw(struct ufs_qcom_host *host, u32 mem_bw, u32 cfg_bw)
842 {
843 struct device *dev = host->hba->dev;
844 int ret;
845
846 ret = icc_set_bw(host->icc_ddr, 0, mem_bw);
847 if (ret < 0) {
848 dev_err(dev, "failed to set bandwidth request: %d\n", ret);
849 return ret;
850 }
851
852 ret = icc_set_bw(host->icc_cpu, 0, cfg_bw);
853 if (ret < 0) {
854 dev_err(dev, "failed to set bandwidth request: %d\n", ret);
855 return ret;
856 }
857
858 return 0;
859 }
860
ufs_qcom_get_bw_table(struct ufs_qcom_host * host)861 static struct __ufs_qcom_bw_table ufs_qcom_get_bw_table(struct ufs_qcom_host *host)
862 {
863 struct ufs_pa_layer_attr *p = &host->dev_req_params;
864 int gear = max_t(u32, p->gear_rx, p->gear_tx);
865 int lane = max_t(u32, p->lane_rx, p->lane_tx);
866
867 if (ufshcd_is_hs_mode(p)) {
868 if (p->hs_rate == PA_HS_MODE_B)
869 return ufs_qcom_bw_table[MODE_HS_RB][gear][lane];
870 else
871 return ufs_qcom_bw_table[MODE_HS_RA][gear][lane];
872 } else {
873 return ufs_qcom_bw_table[MODE_PWM][gear][lane];
874 }
875 }
876
ufs_qcom_icc_update_bw(struct ufs_qcom_host * host)877 static int ufs_qcom_icc_update_bw(struct ufs_qcom_host *host)
878 {
879 struct __ufs_qcom_bw_table bw_table;
880
881 bw_table = ufs_qcom_get_bw_table(host);
882
883 return ufs_qcom_icc_set_bw(host, bw_table.mem_bw, bw_table.cfg_bw);
884 }
885
ufs_qcom_pwr_change_notify(struct ufs_hba * hba,enum ufs_notify_change_status status,struct ufs_pa_layer_attr * dev_max_params,struct ufs_pa_layer_attr * dev_req_params)886 static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba,
887 enum ufs_notify_change_status status,
888 struct ufs_pa_layer_attr *dev_max_params,
889 struct ufs_pa_layer_attr *dev_req_params)
890 {
891 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
892 struct ufs_dev_params ufs_qcom_cap;
893 int ret = 0;
894
895 if (!dev_req_params) {
896 pr_err("%s: incoming dev_req_params is NULL\n", __func__);
897 return -EINVAL;
898 }
899
900 switch (status) {
901 case PRE_CHANGE:
902 ufshcd_init_pwr_dev_param(&ufs_qcom_cap);
903 ufs_qcom_cap.hs_rate = UFS_QCOM_LIMIT_HS_RATE;
904
905 /* This driver only supports symmetic gear setting i.e., hs_tx_gear == hs_rx_gear */
906 ufs_qcom_cap.hs_tx_gear = ufs_qcom_cap.hs_rx_gear = ufs_qcom_get_hs_gear(hba);
907
908 ret = ufshcd_get_pwr_dev_param(&ufs_qcom_cap,
909 dev_max_params,
910 dev_req_params);
911 if (ret) {
912 dev_err(hba->dev, "%s: failed to determine capabilities\n",
913 __func__);
914 return ret;
915 }
916
917 /*
918 * Update hs_gear only when the gears are scaled to a higher value. This is because,
919 * the PHY gear settings are backwards compatible and we only need to change the PHY
920 * settings while scaling to higher gears.
921 */
922 if (dev_req_params->gear_tx > host->hs_gear)
923 host->hs_gear = dev_req_params->gear_tx;
924
925 /* enable the device ref clock before changing to HS mode */
926 if (!ufshcd_is_hs_mode(&hba->pwr_info) &&
927 ufshcd_is_hs_mode(dev_req_params))
928 ufs_qcom_dev_ref_clk_ctrl(host, true);
929
930 if (host->hw_ver.major >= 0x4) {
931 ufshcd_dme_configure_adapt(hba,
932 dev_req_params->gear_tx,
933 PA_INITIAL_ADAPT);
934 }
935 break;
936 case POST_CHANGE:
937 if (ufs_qcom_cfg_timers(hba, dev_req_params->gear_rx,
938 dev_req_params->pwr_rx,
939 dev_req_params->hs_rate, false)) {
940 dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n",
941 __func__);
942 /*
943 * we return error code at the end of the routine,
944 * but continue to configure UFS_PHY_TX_LANE_ENABLE
945 * and bus voting as usual
946 */
947 ret = -EINVAL;
948 }
949
950 /* cache the power mode parameters to use internally */
951 memcpy(&host->dev_req_params,
952 dev_req_params, sizeof(*dev_req_params));
953
954 ufs_qcom_icc_update_bw(host);
955
956 /* disable the device ref clock if entered PWM mode */
957 if (ufshcd_is_hs_mode(&hba->pwr_info) &&
958 !ufshcd_is_hs_mode(dev_req_params))
959 ufs_qcom_dev_ref_clk_ctrl(host, false);
960 break;
961 default:
962 ret = -EINVAL;
963 break;
964 }
965
966 return ret;
967 }
968
ufs_qcom_quirk_host_pa_saveconfigtime(struct ufs_hba * hba)969 static int ufs_qcom_quirk_host_pa_saveconfigtime(struct ufs_hba *hba)
970 {
971 int err;
972 u32 pa_vs_config_reg1;
973
974 err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1),
975 &pa_vs_config_reg1);
976 if (err)
977 return err;
978
979 /* Allow extension of MSB bits of PA_SaveConfigTime attribute */
980 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1),
981 (pa_vs_config_reg1 | (1 << 12)));
982 }
983
ufs_qcom_apply_dev_quirks(struct ufs_hba * hba)984 static int ufs_qcom_apply_dev_quirks(struct ufs_hba *hba)
985 {
986 int err = 0;
987
988 if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME)
989 err = ufs_qcom_quirk_host_pa_saveconfigtime(hba);
990
991 if (hba->dev_info.wmanufacturerid == UFS_VENDOR_WDC)
992 hba->dev_quirks |= UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE;
993
994 return err;
995 }
996
ufs_qcom_get_ufs_hci_version(struct ufs_hba * hba)997 static u32 ufs_qcom_get_ufs_hci_version(struct ufs_hba *hba)
998 {
999 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1000
1001 if (host->hw_ver.major == 0x1)
1002 return ufshci_version(1, 1);
1003 else
1004 return ufshci_version(2, 0);
1005 }
1006
1007 /**
1008 * ufs_qcom_advertise_quirks - advertise the known QCOM UFS controller quirks
1009 * @hba: host controller instance
1010 *
1011 * QCOM UFS host controller might have some non standard behaviours (quirks)
1012 * than what is specified by UFSHCI specification. Advertise all such
1013 * quirks to standard UFS host controller driver so standard takes them into
1014 * account.
1015 */
ufs_qcom_advertise_quirks(struct ufs_hba * hba)1016 static void ufs_qcom_advertise_quirks(struct ufs_hba *hba)
1017 {
1018 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1019
1020 if (host->hw_ver.major == 0x01) {
1021 hba->quirks |= UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS
1022 | UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP
1023 | UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE;
1024
1025 if (host->hw_ver.minor == 0x0001 && host->hw_ver.step == 0x0001)
1026 hba->quirks |= UFSHCD_QUIRK_BROKEN_INTR_AGGR;
1027
1028 hba->quirks |= UFSHCD_QUIRK_BROKEN_LCC;
1029 }
1030
1031 if (host->hw_ver.major == 0x2) {
1032 hba->quirks |= UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION;
1033
1034 if (!ufs_qcom_cap_qunipro(host))
1035 /* Legacy UniPro mode still need following quirks */
1036 hba->quirks |= (UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS
1037 | UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE
1038 | UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP);
1039 }
1040
1041 if (host->hw_ver.major > 0x3)
1042 hba->quirks |= UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH;
1043 }
1044
ufs_qcom_set_caps(struct ufs_hba * hba)1045 static void ufs_qcom_set_caps(struct ufs_hba *hba)
1046 {
1047 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1048
1049 hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
1050 hba->caps |= UFSHCD_CAP_CLK_SCALING | UFSHCD_CAP_WB_WITH_CLK_SCALING;
1051 hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND;
1052 hba->caps |= UFSHCD_CAP_WB_EN;
1053 hba->caps |= UFSHCD_CAP_AGGR_POWER_COLLAPSE;
1054 hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND;
1055
1056 if (host->hw_ver.major >= 0x2) {
1057 host->caps = UFS_QCOM_CAP_QUNIPRO |
1058 UFS_QCOM_CAP_RETAIN_SEC_CFG_AFTER_PWR_COLLAPSE;
1059 }
1060 }
1061
1062 /**
1063 * ufs_qcom_setup_clocks - enables/disable clocks
1064 * @hba: host controller instance
1065 * @on: If true, enable clocks else disable them.
1066 * @status: PRE_CHANGE or POST_CHANGE notify
1067 *
1068 * Return: 0 on success, non-zero on failure.
1069 */
ufs_qcom_setup_clocks(struct ufs_hba * hba,bool on,enum ufs_notify_change_status status)1070 static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on,
1071 enum ufs_notify_change_status status)
1072 {
1073 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1074
1075 /*
1076 * In case ufs_qcom_init() is not yet done, simply ignore.
1077 * This ufs_qcom_setup_clocks() shall be called from
1078 * ufs_qcom_init() after init is done.
1079 */
1080 if (!host)
1081 return 0;
1082
1083 switch (status) {
1084 case PRE_CHANGE:
1085 if (on) {
1086 ufs_qcom_icc_update_bw(host);
1087 } else {
1088 if (!ufs_qcom_is_link_active(hba)) {
1089 /* disable device ref_clk */
1090 ufs_qcom_dev_ref_clk_ctrl(host, false);
1091 }
1092 }
1093 break;
1094 case POST_CHANGE:
1095 if (on) {
1096 /* enable the device ref clock for HS mode*/
1097 if (ufshcd_is_hs_mode(&hba->pwr_info))
1098 ufs_qcom_dev_ref_clk_ctrl(host, true);
1099 } else {
1100 ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MIN][0][0].mem_bw,
1101 ufs_qcom_bw_table[MODE_MIN][0][0].cfg_bw);
1102 }
1103 break;
1104 }
1105
1106 return 0;
1107 }
1108
1109 static int
ufs_qcom_reset_assert(struct reset_controller_dev * rcdev,unsigned long id)1110 ufs_qcom_reset_assert(struct reset_controller_dev *rcdev, unsigned long id)
1111 {
1112 struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev);
1113
1114 ufs_qcom_assert_reset(host->hba);
1115 /* provide 1ms delay to let the reset pulse propagate. */
1116 usleep_range(1000, 1100);
1117 return 0;
1118 }
1119
1120 static int
ufs_qcom_reset_deassert(struct reset_controller_dev * rcdev,unsigned long id)1121 ufs_qcom_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id)
1122 {
1123 struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev);
1124
1125 ufs_qcom_deassert_reset(host->hba);
1126
1127 /*
1128 * after reset deassertion, phy will need all ref clocks,
1129 * voltage, current to settle down before starting serdes.
1130 */
1131 usleep_range(1000, 1100);
1132 return 0;
1133 }
1134
1135 static const struct reset_control_ops ufs_qcom_reset_ops = {
1136 .assert = ufs_qcom_reset_assert,
1137 .deassert = ufs_qcom_reset_deassert,
1138 };
1139
ufs_qcom_icc_init(struct ufs_qcom_host * host)1140 static int ufs_qcom_icc_init(struct ufs_qcom_host *host)
1141 {
1142 struct device *dev = host->hba->dev;
1143 int ret;
1144
1145 host->icc_ddr = devm_of_icc_get(dev, "ufs-ddr");
1146 if (IS_ERR(host->icc_ddr))
1147 return dev_err_probe(dev, PTR_ERR(host->icc_ddr),
1148 "failed to acquire interconnect path\n");
1149
1150 host->icc_cpu = devm_of_icc_get(dev, "cpu-ufs");
1151 if (IS_ERR(host->icc_cpu))
1152 return dev_err_probe(dev, PTR_ERR(host->icc_cpu),
1153 "failed to acquire interconnect path\n");
1154
1155 /*
1156 * Set Maximum bandwidth vote before initializing the UFS controller and
1157 * device. Ideally, a minimal interconnect vote would suffice for the
1158 * initialization, but a max vote would allow faster initialization.
1159 */
1160 ret = ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MAX][0][0].mem_bw,
1161 ufs_qcom_bw_table[MODE_MAX][0][0].cfg_bw);
1162 if (ret < 0)
1163 return dev_err_probe(dev, ret, "failed to set bandwidth request\n");
1164
1165 return 0;
1166 }
1167
1168 /**
1169 * ufs_qcom_init - bind phy with controller
1170 * @hba: host controller instance
1171 *
1172 * Binds PHY with controller and powers up PHY enabling clocks
1173 * and regulators.
1174 *
1175 * Return: -EPROBE_DEFER if binding fails, returns negative error
1176 * on phy power up failure and returns zero on success.
1177 */
ufs_qcom_init(struct ufs_hba * hba)1178 static int ufs_qcom_init(struct ufs_hba *hba)
1179 {
1180 int err;
1181 struct device *dev = hba->dev;
1182 struct platform_device *pdev = to_platform_device(dev);
1183 struct ufs_qcom_host *host;
1184 struct resource *res;
1185 struct ufs_clk_info *clki;
1186
1187 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
1188 if (!host) {
1189 dev_err(dev, "%s: no memory for qcom ufs host\n", __func__);
1190 return -ENOMEM;
1191 }
1192
1193 /* Make a two way bind between the qcom host and the hba */
1194 host->hba = hba;
1195 ufshcd_set_variant(hba, host);
1196
1197 /* Setup the optional reset control of HCI */
1198 host->core_reset = devm_reset_control_get_optional(hba->dev, "rst");
1199 if (IS_ERR(host->core_reset)) {
1200 err = dev_err_probe(dev, PTR_ERR(host->core_reset),
1201 "Failed to get reset control\n");
1202 goto out_variant_clear;
1203 }
1204
1205 /* Fire up the reset controller. Failure here is non-fatal. */
1206 host->rcdev.of_node = dev->of_node;
1207 host->rcdev.ops = &ufs_qcom_reset_ops;
1208 host->rcdev.owner = dev->driver->owner;
1209 host->rcdev.nr_resets = 1;
1210 err = devm_reset_controller_register(dev, &host->rcdev);
1211 if (err)
1212 dev_warn(dev, "Failed to register reset controller\n");
1213
1214 if (!has_acpi_companion(dev)) {
1215 host->generic_phy = devm_phy_get(dev, "ufsphy");
1216 if (IS_ERR(host->generic_phy)) {
1217 err = dev_err_probe(dev, PTR_ERR(host->generic_phy), "Failed to get PHY\n");
1218 goto out_variant_clear;
1219 }
1220 }
1221
1222 err = ufs_qcom_icc_init(host);
1223 if (err)
1224 goto out_variant_clear;
1225
1226 host->device_reset = devm_gpiod_get_optional(dev, "reset",
1227 GPIOD_OUT_HIGH);
1228 if (IS_ERR(host->device_reset)) {
1229 err = PTR_ERR(host->device_reset);
1230 if (err != -EPROBE_DEFER)
1231 dev_err(dev, "failed to acquire reset gpio: %d\n", err);
1232 goto out_variant_clear;
1233 }
1234
1235 ufs_qcom_get_controller_revision(hba, &host->hw_ver.major,
1236 &host->hw_ver.minor, &host->hw_ver.step);
1237
1238 /*
1239 * for newer controllers, device reference clock control bit has
1240 * moved inside UFS controller register address space itself.
1241 */
1242 if (host->hw_ver.major >= 0x02) {
1243 host->dev_ref_clk_ctrl_mmio = hba->mmio_base + REG_UFS_CFG1;
1244 host->dev_ref_clk_en_mask = BIT(26);
1245 } else {
1246 /* "dev_ref_clk_ctrl_mem" is optional resource */
1247 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1248 "dev_ref_clk_ctrl_mem");
1249 if (res) {
1250 host->dev_ref_clk_ctrl_mmio =
1251 devm_ioremap_resource(dev, res);
1252 if (IS_ERR(host->dev_ref_clk_ctrl_mmio))
1253 host->dev_ref_clk_ctrl_mmio = NULL;
1254 host->dev_ref_clk_en_mask = BIT(5);
1255 }
1256 }
1257
1258 list_for_each_entry(clki, &hba->clk_list_head, list) {
1259 if (!strcmp(clki->name, "core_clk_unipro"))
1260 clki->keep_link_active = true;
1261 }
1262
1263 err = ufs_qcom_init_lane_clks(host);
1264 if (err)
1265 goto out_variant_clear;
1266
1267 ufs_qcom_set_caps(hba);
1268 ufs_qcom_advertise_quirks(hba);
1269
1270 err = ufs_qcom_ice_init(host);
1271 if (err)
1272 goto out_variant_clear;
1273
1274 ufs_qcom_setup_clocks(hba, true, POST_CHANGE);
1275
1276 if (hba->dev->id < MAX_UFS_QCOM_HOSTS)
1277 ufs_qcom_hosts[hba->dev->id] = host;
1278
1279 ufs_qcom_get_default_testbus_cfg(host);
1280 err = ufs_qcom_testbus_config(host);
1281 if (err)
1282 /* Failure is non-fatal */
1283 dev_warn(dev, "%s: failed to configure the testbus %d\n",
1284 __func__, err);
1285
1286 /*
1287 * Power up the PHY using the minimum supported gear (UFS_HS_G2).
1288 * Switching to max gear will be performed during reinit if supported.
1289 */
1290 host->hs_gear = UFS_HS_G2;
1291
1292 return 0;
1293
1294 out_variant_clear:
1295 ufshcd_set_variant(hba, NULL);
1296
1297 return err;
1298 }
1299
ufs_qcom_exit(struct ufs_hba * hba)1300 static void ufs_qcom_exit(struct ufs_hba *hba)
1301 {
1302 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1303
1304 ufs_qcom_disable_lane_clks(host);
1305 phy_power_off(host->generic_phy);
1306 phy_exit(host->generic_phy);
1307 }
1308
ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba * hba,u32 clk_cycles)1309 static int ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba *hba,
1310 u32 clk_cycles)
1311 {
1312 int err;
1313 u32 core_clk_ctrl_reg;
1314
1315 if (clk_cycles > DME_VS_CORE_CLK_CTRL_MAX_CORE_CLK_1US_CYCLES_MASK)
1316 return -EINVAL;
1317
1318 err = ufshcd_dme_get(hba,
1319 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1320 &core_clk_ctrl_reg);
1321 if (err)
1322 return err;
1323
1324 core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_MAX_CORE_CLK_1US_CYCLES_MASK;
1325 core_clk_ctrl_reg |= clk_cycles;
1326
1327 /* Clear CORE_CLK_DIV_EN */
1328 core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT;
1329
1330 return ufshcd_dme_set(hba,
1331 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1332 core_clk_ctrl_reg);
1333 }
1334
ufs_qcom_clk_scale_up_pre_change(struct ufs_hba * hba)1335 static int ufs_qcom_clk_scale_up_pre_change(struct ufs_hba *hba)
1336 {
1337 /* nothing to do as of now */
1338 return 0;
1339 }
1340
ufs_qcom_clk_scale_up_post_change(struct ufs_hba * hba)1341 static int ufs_qcom_clk_scale_up_post_change(struct ufs_hba *hba)
1342 {
1343 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1344
1345 if (!ufs_qcom_cap_qunipro(host))
1346 return 0;
1347
1348 /* set unipro core clock cycles to 150 and clear clock divider */
1349 return ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba, 150);
1350 }
1351
ufs_qcom_clk_scale_down_pre_change(struct ufs_hba * hba)1352 static int ufs_qcom_clk_scale_down_pre_change(struct ufs_hba *hba)
1353 {
1354 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1355 int err;
1356 u32 core_clk_ctrl_reg;
1357
1358 if (!ufs_qcom_cap_qunipro(host))
1359 return 0;
1360
1361 err = ufshcd_dme_get(hba,
1362 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1363 &core_clk_ctrl_reg);
1364
1365 /* make sure CORE_CLK_DIV_EN is cleared */
1366 if (!err &&
1367 (core_clk_ctrl_reg & DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT)) {
1368 core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT;
1369 err = ufshcd_dme_set(hba,
1370 UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL),
1371 core_clk_ctrl_reg);
1372 }
1373
1374 return err;
1375 }
1376
ufs_qcom_clk_scale_down_post_change(struct ufs_hba * hba)1377 static int ufs_qcom_clk_scale_down_post_change(struct ufs_hba *hba)
1378 {
1379 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1380
1381 if (!ufs_qcom_cap_qunipro(host))
1382 return 0;
1383
1384 /* set unipro core clock cycles to 75 and clear clock divider */
1385 return ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba, 75);
1386 }
1387
ufs_qcom_clk_scale_notify(struct ufs_hba * hba,bool scale_up,enum ufs_notify_change_status status)1388 static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba,
1389 bool scale_up, enum ufs_notify_change_status status)
1390 {
1391 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1392 struct ufs_pa_layer_attr *dev_req_params = &host->dev_req_params;
1393 int err = 0;
1394
1395 /* check the host controller state before sending hibern8 cmd */
1396 if (!ufshcd_is_hba_active(hba))
1397 return 0;
1398
1399 if (status == PRE_CHANGE) {
1400 err = ufshcd_uic_hibern8_enter(hba);
1401 if (err)
1402 return err;
1403 if (scale_up)
1404 err = ufs_qcom_clk_scale_up_pre_change(hba);
1405 else
1406 err = ufs_qcom_clk_scale_down_pre_change(hba);
1407
1408 if (err) {
1409 ufshcd_uic_hibern8_exit(hba);
1410 return err;
1411 }
1412 } else {
1413 if (scale_up)
1414 err = ufs_qcom_clk_scale_up_post_change(hba);
1415 else
1416 err = ufs_qcom_clk_scale_down_post_change(hba);
1417
1418
1419 if (err) {
1420 ufshcd_uic_hibern8_exit(hba);
1421 return err;
1422 }
1423
1424 ufs_qcom_cfg_timers(hba,
1425 dev_req_params->gear_rx,
1426 dev_req_params->pwr_rx,
1427 dev_req_params->hs_rate,
1428 false);
1429 ufs_qcom_icc_update_bw(host);
1430 ufshcd_uic_hibern8_exit(hba);
1431 }
1432
1433 return 0;
1434 }
1435
ufs_qcom_enable_test_bus(struct ufs_qcom_host * host)1436 static void ufs_qcom_enable_test_bus(struct ufs_qcom_host *host)
1437 {
1438 ufshcd_rmwl(host->hba, UFS_REG_TEST_BUS_EN,
1439 UFS_REG_TEST_BUS_EN, REG_UFS_CFG1);
1440 ufshcd_rmwl(host->hba, TEST_BUS_EN, TEST_BUS_EN, REG_UFS_CFG1);
1441 }
1442
ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host * host)1443 static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host)
1444 {
1445 /* provide a legal default configuration */
1446 host->testbus.select_major = TSTBUS_UNIPRO;
1447 host->testbus.select_minor = 37;
1448 }
1449
ufs_qcom_testbus_cfg_is_ok(struct ufs_qcom_host * host)1450 static bool ufs_qcom_testbus_cfg_is_ok(struct ufs_qcom_host *host)
1451 {
1452 if (host->testbus.select_major >= TSTBUS_MAX) {
1453 dev_err(host->hba->dev,
1454 "%s: UFS_CFG1[TEST_BUS_SEL} may not equal 0x%05X\n",
1455 __func__, host->testbus.select_major);
1456 return false;
1457 }
1458
1459 return true;
1460 }
1461
ufs_qcom_testbus_config(struct ufs_qcom_host * host)1462 int ufs_qcom_testbus_config(struct ufs_qcom_host *host)
1463 {
1464 int reg;
1465 int offset;
1466 u32 mask = TEST_BUS_SUB_SEL_MASK;
1467
1468 if (!host)
1469 return -EINVAL;
1470
1471 if (!ufs_qcom_testbus_cfg_is_ok(host))
1472 return -EPERM;
1473
1474 switch (host->testbus.select_major) {
1475 case TSTBUS_UAWM:
1476 reg = UFS_TEST_BUS_CTRL_0;
1477 offset = 24;
1478 break;
1479 case TSTBUS_UARM:
1480 reg = UFS_TEST_BUS_CTRL_0;
1481 offset = 16;
1482 break;
1483 case TSTBUS_TXUC:
1484 reg = UFS_TEST_BUS_CTRL_0;
1485 offset = 8;
1486 break;
1487 case TSTBUS_RXUC:
1488 reg = UFS_TEST_BUS_CTRL_0;
1489 offset = 0;
1490 break;
1491 case TSTBUS_DFC:
1492 reg = UFS_TEST_BUS_CTRL_1;
1493 offset = 24;
1494 break;
1495 case TSTBUS_TRLUT:
1496 reg = UFS_TEST_BUS_CTRL_1;
1497 offset = 16;
1498 break;
1499 case TSTBUS_TMRLUT:
1500 reg = UFS_TEST_BUS_CTRL_1;
1501 offset = 8;
1502 break;
1503 case TSTBUS_OCSC:
1504 reg = UFS_TEST_BUS_CTRL_1;
1505 offset = 0;
1506 break;
1507 case TSTBUS_WRAPPER:
1508 reg = UFS_TEST_BUS_CTRL_2;
1509 offset = 16;
1510 break;
1511 case TSTBUS_COMBINED:
1512 reg = UFS_TEST_BUS_CTRL_2;
1513 offset = 8;
1514 break;
1515 case TSTBUS_UTP_HCI:
1516 reg = UFS_TEST_BUS_CTRL_2;
1517 offset = 0;
1518 break;
1519 case TSTBUS_UNIPRO:
1520 reg = UFS_UNIPRO_CFG;
1521 offset = 20;
1522 mask = 0xFFF;
1523 break;
1524 /*
1525 * No need for a default case, since
1526 * ufs_qcom_testbus_cfg_is_ok() checks that the configuration
1527 * is legal
1528 */
1529 }
1530 mask <<= offset;
1531 ufshcd_rmwl(host->hba, TEST_BUS_SEL,
1532 (u32)host->testbus.select_major << 19,
1533 REG_UFS_CFG1);
1534 ufshcd_rmwl(host->hba, mask,
1535 (u32)host->testbus.select_minor << offset,
1536 reg);
1537 ufs_qcom_enable_test_bus(host);
1538 /*
1539 * Make sure the test bus configuration is
1540 * committed before returning.
1541 */
1542 mb();
1543
1544 return 0;
1545 }
1546
ufs_qcom_dump_dbg_regs(struct ufs_hba * hba)1547 static void ufs_qcom_dump_dbg_regs(struct ufs_hba *hba)
1548 {
1549 u32 reg;
1550 struct ufs_qcom_host *host;
1551
1552 host = ufshcd_get_variant(hba);
1553
1554 ufshcd_dump_regs(hba, REG_UFS_SYS1CLK_1US, 16 * 4,
1555 "HCI Vendor Specific Registers ");
1556
1557 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_REG_OCSC);
1558 ufshcd_dump_regs(hba, reg, 44 * 4, "UFS_UFS_DBG_RD_REG_OCSC ");
1559
1560 reg = ufshcd_readl(hba, REG_UFS_CFG1);
1561 reg |= UTP_DBG_RAMS_EN;
1562 ufshcd_writel(hba, reg, REG_UFS_CFG1);
1563
1564 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_EDTL_RAM);
1565 ufshcd_dump_regs(hba, reg, 32 * 4, "UFS_UFS_DBG_RD_EDTL_RAM ");
1566
1567 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_DESC_RAM);
1568 ufshcd_dump_regs(hba, reg, 128 * 4, "UFS_UFS_DBG_RD_DESC_RAM ");
1569
1570 reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_PRDT_RAM);
1571 ufshcd_dump_regs(hba, reg, 64 * 4, "UFS_UFS_DBG_RD_PRDT_RAM ");
1572
1573 /* clear bit 17 - UTP_DBG_RAMS_EN */
1574 ufshcd_rmwl(hba, UTP_DBG_RAMS_EN, 0, REG_UFS_CFG1);
1575
1576 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UAWM);
1577 ufshcd_dump_regs(hba, reg, 4 * 4, "UFS_DBG_RD_REG_UAWM ");
1578
1579 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UARM);
1580 ufshcd_dump_regs(hba, reg, 4 * 4, "UFS_DBG_RD_REG_UARM ");
1581
1582 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TXUC);
1583 ufshcd_dump_regs(hba, reg, 48 * 4, "UFS_DBG_RD_REG_TXUC ");
1584
1585 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_RXUC);
1586 ufshcd_dump_regs(hba, reg, 27 * 4, "UFS_DBG_RD_REG_RXUC ");
1587
1588 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_DFC);
1589 ufshcd_dump_regs(hba, reg, 19 * 4, "UFS_DBG_RD_REG_DFC ");
1590
1591 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TRLUT);
1592 ufshcd_dump_regs(hba, reg, 34 * 4, "UFS_DBG_RD_REG_TRLUT ");
1593
1594 reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TMRLUT);
1595 ufshcd_dump_regs(hba, reg, 9 * 4, "UFS_DBG_RD_REG_TMRLUT ");
1596 }
1597
1598 /**
1599 * ufs_qcom_device_reset() - toggle the (optional) device reset line
1600 * @hba: per-adapter instance
1601 *
1602 * Toggles the (optional) reset line to reset the attached device.
1603 */
ufs_qcom_device_reset(struct ufs_hba * hba)1604 static int ufs_qcom_device_reset(struct ufs_hba *hba)
1605 {
1606 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1607
1608 /* reset gpio is optional */
1609 if (!host->device_reset)
1610 return -EOPNOTSUPP;
1611
1612 /*
1613 * The UFS device shall detect reset pulses of 1us, sleep for 10us to
1614 * be on the safe side.
1615 */
1616 ufs_qcom_device_reset_ctrl(hba, true);
1617 usleep_range(10, 15);
1618
1619 ufs_qcom_device_reset_ctrl(hba, false);
1620 usleep_range(10, 15);
1621
1622 return 0;
1623 }
1624
1625 #if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND)
ufs_qcom_config_scaling_param(struct ufs_hba * hba,struct devfreq_dev_profile * p,struct devfreq_simple_ondemand_data * d)1626 static void ufs_qcom_config_scaling_param(struct ufs_hba *hba,
1627 struct devfreq_dev_profile *p,
1628 struct devfreq_simple_ondemand_data *d)
1629 {
1630 p->polling_ms = 60;
1631 p->timer = DEVFREQ_TIMER_DELAYED;
1632 d->upthreshold = 70;
1633 d->downdifferential = 5;
1634 }
1635 #else
ufs_qcom_config_scaling_param(struct ufs_hba * hba,struct devfreq_dev_profile * p,struct devfreq_simple_ondemand_data * data)1636 static void ufs_qcom_config_scaling_param(struct ufs_hba *hba,
1637 struct devfreq_dev_profile *p,
1638 struct devfreq_simple_ondemand_data *data)
1639 {
1640 }
1641 #endif
1642
1643 /* Resources */
1644 static const struct ufshcd_res_info ufs_res_info[RES_MAX] = {
1645 {.name = "ufs_mem",},
1646 {.name = "mcq",},
1647 /* Submission Queue DAO */
1648 {.name = "mcq_sqd",},
1649 /* Submission Queue Interrupt Status */
1650 {.name = "mcq_sqis",},
1651 /* Completion Queue DAO */
1652 {.name = "mcq_cqd",},
1653 /* Completion Queue Interrupt Status */
1654 {.name = "mcq_cqis",},
1655 /* MCQ vendor specific */
1656 {.name = "mcq_vs",},
1657 };
1658
ufs_qcom_mcq_config_resource(struct ufs_hba * hba)1659 static int ufs_qcom_mcq_config_resource(struct ufs_hba *hba)
1660 {
1661 struct platform_device *pdev = to_platform_device(hba->dev);
1662 struct ufshcd_res_info *res;
1663 struct resource *res_mem, *res_mcq;
1664 int i, ret = 0;
1665
1666 memcpy(hba->res, ufs_res_info, sizeof(ufs_res_info));
1667
1668 for (i = 0; i < RES_MAX; i++) {
1669 res = &hba->res[i];
1670 res->resource = platform_get_resource_byname(pdev,
1671 IORESOURCE_MEM,
1672 res->name);
1673 if (!res->resource) {
1674 dev_info(hba->dev, "Resource %s not provided\n", res->name);
1675 if (i == RES_UFS)
1676 return -ENODEV;
1677 continue;
1678 } else if (i == RES_UFS) {
1679 res_mem = res->resource;
1680 res->base = hba->mmio_base;
1681 continue;
1682 }
1683
1684 res->base = devm_ioremap_resource(hba->dev, res->resource);
1685 if (IS_ERR(res->base)) {
1686 dev_err(hba->dev, "Failed to map res %s, err=%d\n",
1687 res->name, (int)PTR_ERR(res->base));
1688 ret = PTR_ERR(res->base);
1689 res->base = NULL;
1690 return ret;
1691 }
1692 }
1693
1694 /* MCQ resource provided in DT */
1695 res = &hba->res[RES_MCQ];
1696 /* Bail if MCQ resource is provided */
1697 if (res->base)
1698 goto out;
1699
1700 /* Explicitly allocate MCQ resource from ufs_mem */
1701 res_mcq = devm_kzalloc(hba->dev, sizeof(*res_mcq), GFP_KERNEL);
1702 if (!res_mcq)
1703 return -ENOMEM;
1704
1705 res_mcq->start = res_mem->start +
1706 MCQ_SQATTR_OFFSET(hba->mcq_capabilities);
1707 res_mcq->end = res_mcq->start + hba->nr_hw_queues * MCQ_QCFG_SIZE - 1;
1708 res_mcq->flags = res_mem->flags;
1709 res_mcq->name = "mcq";
1710
1711 ret = insert_resource(&iomem_resource, res_mcq);
1712 if (ret) {
1713 dev_err(hba->dev, "Failed to insert MCQ resource, err=%d\n",
1714 ret);
1715 return ret;
1716 }
1717
1718 res->base = devm_ioremap_resource(hba->dev, res_mcq);
1719 if (IS_ERR(res->base)) {
1720 dev_err(hba->dev, "MCQ registers mapping failed, err=%d\n",
1721 (int)PTR_ERR(res->base));
1722 ret = PTR_ERR(res->base);
1723 goto ioremap_err;
1724 }
1725
1726 out:
1727 hba->mcq_base = res->base;
1728 return 0;
1729 ioremap_err:
1730 res->base = NULL;
1731 remove_resource(res_mcq);
1732 return ret;
1733 }
1734
ufs_qcom_op_runtime_config(struct ufs_hba * hba)1735 static int ufs_qcom_op_runtime_config(struct ufs_hba *hba)
1736 {
1737 struct ufshcd_res_info *mem_res, *sqdao_res;
1738 struct ufshcd_mcq_opr_info_t *opr;
1739 int i;
1740
1741 mem_res = &hba->res[RES_UFS];
1742 sqdao_res = &hba->res[RES_MCQ_SQD];
1743
1744 if (!mem_res->base || !sqdao_res->base)
1745 return -EINVAL;
1746
1747 for (i = 0; i < OPR_MAX; i++) {
1748 opr = &hba->mcq_opr[i];
1749 opr->offset = sqdao_res->resource->start -
1750 mem_res->resource->start + 0x40 * i;
1751 opr->stride = 0x100;
1752 opr->base = sqdao_res->base + 0x40 * i;
1753 }
1754
1755 return 0;
1756 }
1757
ufs_qcom_get_hba_mac(struct ufs_hba * hba)1758 static int ufs_qcom_get_hba_mac(struct ufs_hba *hba)
1759 {
1760 /* Qualcomm HC supports up to 64 */
1761 return MAX_SUPP_MAC;
1762 }
1763
ufs_qcom_get_outstanding_cqs(struct ufs_hba * hba,unsigned long * ocqs)1764 static int ufs_qcom_get_outstanding_cqs(struct ufs_hba *hba,
1765 unsigned long *ocqs)
1766 {
1767 struct ufshcd_res_info *mcq_vs_res = &hba->res[RES_MCQ_VS];
1768
1769 if (!mcq_vs_res->base)
1770 return -EINVAL;
1771
1772 *ocqs = readl(mcq_vs_res->base + UFS_MEM_CQIS_VS);
1773
1774 return 0;
1775 }
1776
ufs_qcom_write_msi_msg(struct msi_desc * desc,struct msi_msg * msg)1777 static void ufs_qcom_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
1778 {
1779 struct device *dev = msi_desc_to_dev(desc);
1780 struct ufs_hba *hba = dev_get_drvdata(dev);
1781
1782 ufshcd_mcq_config_esi(hba, msg);
1783 }
1784
ufs_qcom_mcq_esi_handler(int irq,void * data)1785 static irqreturn_t ufs_qcom_mcq_esi_handler(int irq, void *data)
1786 {
1787 struct msi_desc *desc = data;
1788 struct device *dev = msi_desc_to_dev(desc);
1789 struct ufs_hba *hba = dev_get_drvdata(dev);
1790 u32 id = desc->msi_index;
1791 struct ufs_hw_queue *hwq = &hba->uhq[id];
1792
1793 ufshcd_mcq_write_cqis(hba, 0x1, id);
1794 ufshcd_mcq_poll_cqe_lock(hba, hwq);
1795
1796 return IRQ_HANDLED;
1797 }
1798
ufs_qcom_config_esi(struct ufs_hba * hba)1799 static int ufs_qcom_config_esi(struct ufs_hba *hba)
1800 {
1801 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1802 struct msi_desc *desc;
1803 struct msi_desc *failed_desc = NULL;
1804 int nr_irqs, ret;
1805
1806 if (host->esi_enabled)
1807 return 0;
1808
1809 /*
1810 * 1. We only handle CQs as of now.
1811 * 2. Poll queues do not need ESI.
1812 */
1813 nr_irqs = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL];
1814 ret = platform_msi_domain_alloc_irqs(hba->dev, nr_irqs,
1815 ufs_qcom_write_msi_msg);
1816 if (ret) {
1817 dev_err(hba->dev, "Failed to request Platform MSI %d\n", ret);
1818 goto out;
1819 }
1820
1821 msi_lock_descs(hba->dev);
1822 msi_for_each_desc(desc, hba->dev, MSI_DESC_ALL) {
1823 ret = devm_request_irq(hba->dev, desc->irq,
1824 ufs_qcom_mcq_esi_handler,
1825 IRQF_SHARED, "qcom-mcq-esi", desc);
1826 if (ret) {
1827 dev_err(hba->dev, "%s: Fail to request IRQ for %d, err = %d\n",
1828 __func__, desc->irq, ret);
1829 failed_desc = desc;
1830 break;
1831 }
1832 }
1833 msi_unlock_descs(hba->dev);
1834
1835 if (ret) {
1836 /* Rewind */
1837 msi_lock_descs(hba->dev);
1838 msi_for_each_desc(desc, hba->dev, MSI_DESC_ALL) {
1839 if (desc == failed_desc)
1840 break;
1841 devm_free_irq(hba->dev, desc->irq, hba);
1842 }
1843 msi_unlock_descs(hba->dev);
1844 platform_msi_domain_free_irqs(hba->dev);
1845 } else {
1846 if (host->hw_ver.major == 6 && host->hw_ver.minor == 0 &&
1847 host->hw_ver.step == 0) {
1848 ufshcd_writel(hba,
1849 ufshcd_readl(hba, REG_UFS_CFG3) | 0x1F000,
1850 REG_UFS_CFG3);
1851 }
1852 ufshcd_mcq_enable_esi(hba);
1853 }
1854
1855 out:
1856 if (!ret)
1857 host->esi_enabled = true;
1858
1859 return ret;
1860 }
1861
1862 /*
1863 * struct ufs_hba_qcom_vops - UFS QCOM specific variant operations
1864 *
1865 * The variant operations configure the necessary controller and PHY
1866 * handshake during initialization.
1867 */
1868 static const struct ufs_hba_variant_ops ufs_hba_qcom_vops = {
1869 .name = "qcom",
1870 .init = ufs_qcom_init,
1871 .exit = ufs_qcom_exit,
1872 .get_ufs_hci_version = ufs_qcom_get_ufs_hci_version,
1873 .clk_scale_notify = ufs_qcom_clk_scale_notify,
1874 .setup_clocks = ufs_qcom_setup_clocks,
1875 .hce_enable_notify = ufs_qcom_hce_enable_notify,
1876 .link_startup_notify = ufs_qcom_link_startup_notify,
1877 .pwr_change_notify = ufs_qcom_pwr_change_notify,
1878 .apply_dev_quirks = ufs_qcom_apply_dev_quirks,
1879 .suspend = ufs_qcom_suspend,
1880 .resume = ufs_qcom_resume,
1881 .dbg_register_dump = ufs_qcom_dump_dbg_regs,
1882 .device_reset = ufs_qcom_device_reset,
1883 .config_scaling_param = ufs_qcom_config_scaling_param,
1884 .program_key = ufs_qcom_ice_program_key,
1885 .mcq_config_resource = ufs_qcom_mcq_config_resource,
1886 .get_hba_mac = ufs_qcom_get_hba_mac,
1887 .op_runtime_config = ufs_qcom_op_runtime_config,
1888 .get_outstanding_cqs = ufs_qcom_get_outstanding_cqs,
1889 .config_esi = ufs_qcom_config_esi,
1890 };
1891
1892 /**
1893 * ufs_qcom_probe - probe routine of the driver
1894 * @pdev: pointer to Platform device handle
1895 *
1896 * Return: zero for success and non-zero for failure.
1897 */
ufs_qcom_probe(struct platform_device * pdev)1898 static int ufs_qcom_probe(struct platform_device *pdev)
1899 {
1900 int err;
1901 struct device *dev = &pdev->dev;
1902
1903 /* Perform generic probe */
1904 err = ufshcd_pltfrm_init(pdev, &ufs_hba_qcom_vops);
1905 if (err)
1906 return dev_err_probe(dev, err, "ufshcd_pltfrm_init() failed\n");
1907
1908 return 0;
1909 }
1910
1911 /**
1912 * ufs_qcom_remove - set driver_data of the device to NULL
1913 * @pdev: pointer to platform device handle
1914 *
1915 * Always returns 0
1916 */
ufs_qcom_remove(struct platform_device * pdev)1917 static int ufs_qcom_remove(struct platform_device *pdev)
1918 {
1919 struct ufs_hba *hba = platform_get_drvdata(pdev);
1920 struct ufs_qcom_host *host = ufshcd_get_variant(hba);
1921
1922 pm_runtime_get_sync(&(pdev)->dev);
1923 ufshcd_remove(hba);
1924 if (host->esi_enabled)
1925 platform_msi_domain_free_irqs(hba->dev);
1926 return 0;
1927 }
1928
1929 static const struct of_device_id ufs_qcom_of_match[] __maybe_unused = {
1930 { .compatible = "qcom,ufshc"},
1931 {},
1932 };
1933 MODULE_DEVICE_TABLE(of, ufs_qcom_of_match);
1934
1935 #ifdef CONFIG_ACPI
1936 static const struct acpi_device_id ufs_qcom_acpi_match[] = {
1937 { "QCOM24A5" },
1938 { },
1939 };
1940 MODULE_DEVICE_TABLE(acpi, ufs_qcom_acpi_match);
1941 #endif
1942
1943 static const struct dev_pm_ops ufs_qcom_pm_ops = {
1944 SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL)
1945 .prepare = ufshcd_suspend_prepare,
1946 .complete = ufshcd_resume_complete,
1947 #ifdef CONFIG_PM_SLEEP
1948 .suspend = ufshcd_system_suspend,
1949 .resume = ufshcd_system_resume,
1950 .freeze = ufshcd_system_freeze,
1951 .restore = ufshcd_system_restore,
1952 .thaw = ufshcd_system_thaw,
1953 #endif
1954 };
1955
1956 static struct platform_driver ufs_qcom_pltform = {
1957 .probe = ufs_qcom_probe,
1958 .remove = ufs_qcom_remove,
1959 .driver = {
1960 .name = "ufshcd-qcom",
1961 .pm = &ufs_qcom_pm_ops,
1962 .of_match_table = of_match_ptr(ufs_qcom_of_match),
1963 .acpi_match_table = ACPI_PTR(ufs_qcom_acpi_match),
1964 },
1965 };
1966 module_platform_driver(ufs_qcom_pltform);
1967
1968 MODULE_LICENSE("GPL v2");
1969