xref: /openbmc/linux/drivers/gpu/drm/radeon/ci_dpm.c (revision 20c3dffd)
1 /*
2  * Copyright 2013 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/pci.h>
26 #include <linux/seq_file.h>
27 
28 #include "atom.h"
29 #include "ci_dpm.h"
30 #include "cik.h"
31 #include "cikd.h"
32 #include "r600_dpm.h"
33 #include "radeon.h"
34 #include "radeon_asic.h"
35 #include "radeon_ucode.h"
36 #include "si_dpm.h"
37 
38 #define MC_CG_ARB_FREQ_F0           0x0a
39 #define MC_CG_ARB_FREQ_F1           0x0b
40 #define MC_CG_ARB_FREQ_F2           0x0c
41 #define MC_CG_ARB_FREQ_F3           0x0d
42 
43 #define SMC_RAM_END 0x40000
44 
45 #define VOLTAGE_SCALE               4
46 #define VOLTAGE_VID_OFFSET_SCALE1    625
47 #define VOLTAGE_VID_OFFSET_SCALE2    100
48 
49 static const struct ci_pt_defaults defaults_hawaii_xt =
50 {
51 	1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0xB0000,
52 	{ 0x2E,  0x00,  0x00,  0x88,  0x00,  0x00,  0x72,  0x60,  0x51,  0xA7,  0x79,  0x6B,  0x90,  0xBD,  0x79  },
53 	{ 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 }
54 };
55 
56 static const struct ci_pt_defaults defaults_hawaii_pro =
57 {
58 	1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0x65062,
59 	{ 0x2E,  0x00,  0x00,  0x88,  0x00,  0x00,  0x72,  0x60,  0x51,  0xA7,  0x79,  0x6B,  0x90,  0xBD,  0x79  },
60 	{ 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 }
61 };
62 
63 static const struct ci_pt_defaults defaults_bonaire_xt =
64 {
65 	1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000,
66 	{ 0x79,  0x253, 0x25D, 0xAE,  0x72,  0x80,  0x83,  0x86,  0x6F,  0xC8,  0xC9,  0xC9,  0x2F,  0x4D,  0x61  },
67 	{ 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 }
68 };
69 
70 static const struct ci_pt_defaults defaults_saturn_xt =
71 {
72 	1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x70000,
73 	{ 0x8C,  0x247, 0x249, 0xA6,  0x80,  0x81,  0x8B,  0x89,  0x86,  0xC9,  0xCA,  0xC9,  0x4D,  0x4D,  0x4D  },
74 	{ 0x187, 0x187, 0x187, 0x1C7, 0x1C7, 0x1C7, 0x210, 0x210, 0x210, 0x266, 0x266, 0x266, 0x2C9, 0x2C9, 0x2C9 }
75 };
76 
77 static const struct ci_pt_config_reg didt_config_ci[] =
78 {
79 	{ 0x10, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
80 	{ 0x10, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
81 	{ 0x10, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
82 	{ 0x10, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
83 	{ 0x11, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
84 	{ 0x11, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
85 	{ 0x11, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
86 	{ 0x11, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
87 	{ 0x12, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
88 	{ 0x12, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
89 	{ 0x12, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
90 	{ 0x12, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
91 	{ 0x2, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
92 	{ 0x2, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
93 	{ 0x2, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
94 	{ 0x1, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
95 	{ 0x1, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
96 	{ 0x0, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
97 	{ 0x30, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
98 	{ 0x30, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
99 	{ 0x30, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
100 	{ 0x30, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
101 	{ 0x31, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
102 	{ 0x31, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
103 	{ 0x31, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
104 	{ 0x31, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
105 	{ 0x32, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
106 	{ 0x32, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
107 	{ 0x32, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
108 	{ 0x32, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
109 	{ 0x22, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
110 	{ 0x22, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
111 	{ 0x22, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
112 	{ 0x21, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
113 	{ 0x21, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
114 	{ 0x20, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
115 	{ 0x50, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
116 	{ 0x50, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
117 	{ 0x50, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
118 	{ 0x50, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
119 	{ 0x51, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
120 	{ 0x51, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
121 	{ 0x51, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
122 	{ 0x51, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
123 	{ 0x52, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
124 	{ 0x52, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
125 	{ 0x52, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
126 	{ 0x52, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
127 	{ 0x42, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
128 	{ 0x42, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
129 	{ 0x42, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
130 	{ 0x41, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
131 	{ 0x41, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
132 	{ 0x40, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
133 	{ 0x70, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
134 	{ 0x70, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
135 	{ 0x70, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
136 	{ 0x70, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
137 	{ 0x71, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
138 	{ 0x71, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
139 	{ 0x71, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
140 	{ 0x71, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
141 	{ 0x72, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
142 	{ 0x72, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
143 	{ 0x72, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
144 	{ 0x72, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
145 	{ 0x62, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
146 	{ 0x62, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
147 	{ 0x62, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
148 	{ 0x61, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
149 	{ 0x61, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
150 	{ 0x60, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
151 	{ 0xFFFFFFFF }
152 };
153 
154 extern u8 rv770_get_memory_module_index(struct radeon_device *rdev);
155 extern int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
156 				       u32 arb_freq_src, u32 arb_freq_dest);
157 static int ci_get_std_voltage_value_sidd(struct radeon_device *rdev,
158 					 struct atom_voltage_table_entry *voltage_table,
159 					 u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd);
160 static int ci_set_power_limit(struct radeon_device *rdev, u32 n);
161 static int ci_set_overdrive_target_tdp(struct radeon_device *rdev,
162 				       u32 target_tdp);
163 static int ci_update_uvd_dpm(struct radeon_device *rdev, bool gate);
164 
165 static PPSMC_Result ci_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg);
166 static PPSMC_Result ci_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
167 						      PPSMC_Msg msg, u32 parameter);
168 
169 static void ci_thermal_start_smc_fan_control(struct radeon_device *rdev);
170 static void ci_fan_ctrl_set_default_mode(struct radeon_device *rdev);
171 
ci_get_pi(struct radeon_device * rdev)172 static struct ci_power_info *ci_get_pi(struct radeon_device *rdev)
173 {
174 	struct ci_power_info *pi = rdev->pm.dpm.priv;
175 
176 	return pi;
177 }
178 
ci_get_ps(struct radeon_ps * rps)179 static struct ci_ps *ci_get_ps(struct radeon_ps *rps)
180 {
181 	struct ci_ps *ps = rps->ps_priv;
182 
183 	return ps;
184 }
185 
ci_initialize_powertune_defaults(struct radeon_device * rdev)186 static void ci_initialize_powertune_defaults(struct radeon_device *rdev)
187 {
188 	struct ci_power_info *pi = ci_get_pi(rdev);
189 
190 	switch (rdev->pdev->device) {
191 	case 0x6649:
192 	case 0x6650:
193 	case 0x6651:
194 	case 0x6658:
195 	case 0x665C:
196 	case 0x665D:
197 	default:
198 		pi->powertune_defaults = &defaults_bonaire_xt;
199 		break;
200 	case 0x6640:
201 	case 0x6641:
202 	case 0x6646:
203 	case 0x6647:
204 		pi->powertune_defaults = &defaults_saturn_xt;
205 		break;
206 	case 0x67B8:
207 	case 0x67B0:
208 		pi->powertune_defaults = &defaults_hawaii_xt;
209 		break;
210 	case 0x67BA:
211 	case 0x67B1:
212 		pi->powertune_defaults = &defaults_hawaii_pro;
213 		break;
214 	case 0x67A0:
215 	case 0x67A1:
216 	case 0x67A2:
217 	case 0x67A8:
218 	case 0x67A9:
219 	case 0x67AA:
220 	case 0x67B9:
221 	case 0x67BE:
222 		pi->powertune_defaults = &defaults_bonaire_xt;
223 		break;
224 	}
225 
226 	pi->dte_tj_offset = 0;
227 
228 	pi->caps_power_containment = true;
229 	pi->caps_cac = false;
230 	pi->caps_sq_ramping = false;
231 	pi->caps_db_ramping = false;
232 	pi->caps_td_ramping = false;
233 	pi->caps_tcp_ramping = false;
234 
235 	if (pi->caps_power_containment) {
236 		pi->caps_cac = true;
237 		if (rdev->family == CHIP_HAWAII)
238 			pi->enable_bapm_feature = false;
239 		else
240 			pi->enable_bapm_feature = true;
241 		pi->enable_tdc_limit_feature = true;
242 		pi->enable_pkg_pwr_tracking_feature = true;
243 	}
244 }
245 
ci_convert_to_vid(u16 vddc)246 static u8 ci_convert_to_vid(u16 vddc)
247 {
248 	return (6200 - (vddc * VOLTAGE_SCALE)) / 25;
249 }
250 
ci_populate_bapm_vddc_vid_sidd(struct radeon_device * rdev)251 static int ci_populate_bapm_vddc_vid_sidd(struct radeon_device *rdev)
252 {
253 	struct ci_power_info *pi = ci_get_pi(rdev);
254 	u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd;
255 	u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd;
256 	u8 *hi2_vid = pi->smc_powertune_table.BapmVddCVidHiSidd2;
257 	u32 i;
258 
259 	if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries == NULL)
260 		return -EINVAL;
261 	if (rdev->pm.dpm.dyn_state.cac_leakage_table.count > 8)
262 		return -EINVAL;
263 	if (rdev->pm.dpm.dyn_state.cac_leakage_table.count !=
264 	    rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count)
265 		return -EINVAL;
266 
267 	for (i = 0; i < rdev->pm.dpm.dyn_state.cac_leakage_table.count; i++) {
268 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) {
269 			lo_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc1);
270 			hi_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc2);
271 			hi2_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc3);
272 		} else {
273 			lo_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc);
274 			hi_vid[i] = ci_convert_to_vid((u16)rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].leakage);
275 		}
276 	}
277 	return 0;
278 }
279 
ci_populate_vddc_vid(struct radeon_device * rdev)280 static int ci_populate_vddc_vid(struct radeon_device *rdev)
281 {
282 	struct ci_power_info *pi = ci_get_pi(rdev);
283 	u8 *vid = pi->smc_powertune_table.VddCVid;
284 	u32 i;
285 
286 	if (pi->vddc_voltage_table.count > 8)
287 		return -EINVAL;
288 
289 	for (i = 0; i < pi->vddc_voltage_table.count; i++)
290 		vid[i] = ci_convert_to_vid(pi->vddc_voltage_table.entries[i].value);
291 
292 	return 0;
293 }
294 
ci_populate_svi_load_line(struct radeon_device * rdev)295 static int ci_populate_svi_load_line(struct radeon_device *rdev)
296 {
297 	struct ci_power_info *pi = ci_get_pi(rdev);
298 	const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
299 
300 	pi->smc_powertune_table.SviLoadLineEn = pt_defaults->svi_load_line_en;
301 	pi->smc_powertune_table.SviLoadLineVddC = pt_defaults->svi_load_line_vddc;
302 	pi->smc_powertune_table.SviLoadLineTrimVddC = 3;
303 	pi->smc_powertune_table.SviLoadLineOffsetVddC = 0;
304 
305 	return 0;
306 }
307 
ci_populate_tdc_limit(struct radeon_device * rdev)308 static int ci_populate_tdc_limit(struct radeon_device *rdev)
309 {
310 	struct ci_power_info *pi = ci_get_pi(rdev);
311 	const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
312 	u16 tdc_limit;
313 
314 	tdc_limit = rdev->pm.dpm.dyn_state.cac_tdp_table->tdc * 256;
315 	pi->smc_powertune_table.TDC_VDDC_PkgLimit = cpu_to_be16(tdc_limit);
316 	pi->smc_powertune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
317 		pt_defaults->tdc_vddc_throttle_release_limit_perc;
318 	pi->smc_powertune_table.TDC_MAWt = pt_defaults->tdc_mawt;
319 
320 	return 0;
321 }
322 
ci_populate_dw8(struct radeon_device * rdev)323 static int ci_populate_dw8(struct radeon_device *rdev)
324 {
325 	struct ci_power_info *pi = ci_get_pi(rdev);
326 	const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
327 	int ret;
328 
329 	ret = ci_read_smc_sram_dword(rdev,
330 				     SMU7_FIRMWARE_HEADER_LOCATION +
331 				     offsetof(SMU7_Firmware_Header, PmFuseTable) +
332 				     offsetof(SMU7_Discrete_PmFuses, TdcWaterfallCtl),
333 				     (u32 *)&pi->smc_powertune_table.TdcWaterfallCtl,
334 				     pi->sram_end);
335 	if (ret)
336 		return -EINVAL;
337 	else
338 		pi->smc_powertune_table.TdcWaterfallCtl = pt_defaults->tdc_waterfall_ctl;
339 
340 	return 0;
341 }
342 
ci_populate_fuzzy_fan(struct radeon_device * rdev)343 static int ci_populate_fuzzy_fan(struct radeon_device *rdev)
344 {
345 	struct ci_power_info *pi = ci_get_pi(rdev);
346 
347 	if ((rdev->pm.dpm.fan.fan_output_sensitivity & (1 << 15)) ||
348 	    (rdev->pm.dpm.fan.fan_output_sensitivity == 0))
349 		rdev->pm.dpm.fan.fan_output_sensitivity =
350 			rdev->pm.dpm.fan.default_fan_output_sensitivity;
351 
352 	pi->smc_powertune_table.FuzzyFan_PwmSetDelta =
353 		cpu_to_be16(rdev->pm.dpm.fan.fan_output_sensitivity);
354 
355 	return 0;
356 }
357 
ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(struct radeon_device * rdev)358 static int ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(struct radeon_device *rdev)
359 {
360 	struct ci_power_info *pi = ci_get_pi(rdev);
361 	u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd;
362 	u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd;
363 	int i, min, max;
364 
365 	min = max = hi_vid[0];
366 	for (i = 0; i < 8; i++) {
367 		if (0 != hi_vid[i]) {
368 			if (min > hi_vid[i])
369 				min = hi_vid[i];
370 			if (max < hi_vid[i])
371 				max = hi_vid[i];
372 		}
373 
374 		if (0 != lo_vid[i]) {
375 			if (min > lo_vid[i])
376 				min = lo_vid[i];
377 			if (max < lo_vid[i])
378 				max = lo_vid[i];
379 		}
380 	}
381 
382 	if ((min == 0) || (max == 0))
383 		return -EINVAL;
384 	pi->smc_powertune_table.GnbLPMLMaxVid = (u8)max;
385 	pi->smc_powertune_table.GnbLPMLMinVid = (u8)min;
386 
387 	return 0;
388 }
389 
ci_populate_bapm_vddc_base_leakage_sidd(struct radeon_device * rdev)390 static int ci_populate_bapm_vddc_base_leakage_sidd(struct radeon_device *rdev)
391 {
392 	struct ci_power_info *pi = ci_get_pi(rdev);
393 	u16 hi_sidd, lo_sidd;
394 	struct radeon_cac_tdp_table *cac_tdp_table =
395 		rdev->pm.dpm.dyn_state.cac_tdp_table;
396 
397 	hi_sidd = cac_tdp_table->high_cac_leakage / 100 * 256;
398 	lo_sidd = cac_tdp_table->low_cac_leakage / 100 * 256;
399 
400 	pi->smc_powertune_table.BapmVddCBaseLeakageHiSidd = cpu_to_be16(hi_sidd);
401 	pi->smc_powertune_table.BapmVddCBaseLeakageLoSidd = cpu_to_be16(lo_sidd);
402 
403 	return 0;
404 }
405 
ci_populate_bapm_parameters_in_dpm_table(struct radeon_device * rdev)406 static int ci_populate_bapm_parameters_in_dpm_table(struct radeon_device *rdev)
407 {
408 	struct ci_power_info *pi = ci_get_pi(rdev);
409 	const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
410 	SMU7_Discrete_DpmTable  *dpm_table = &pi->smc_state_table;
411 	struct radeon_cac_tdp_table *cac_tdp_table =
412 		rdev->pm.dpm.dyn_state.cac_tdp_table;
413 	struct radeon_ppm_table *ppm = rdev->pm.dpm.dyn_state.ppm_table;
414 	int i, j, k;
415 	const u16 *def1;
416 	const u16 *def2;
417 
418 	dpm_table->DefaultTdp = cac_tdp_table->tdp * 256;
419 	dpm_table->TargetTdp = cac_tdp_table->configurable_tdp * 256;
420 
421 	dpm_table->DTETjOffset = (u8)pi->dte_tj_offset;
422 	dpm_table->GpuTjMax =
423 		(u8)(pi->thermal_temp_setting.temperature_high / 1000);
424 	dpm_table->GpuTjHyst = 8;
425 
426 	dpm_table->DTEAmbientTempBase = pt_defaults->dte_ambient_temp_base;
427 
428 	if (ppm) {
429 		dpm_table->PPM_PkgPwrLimit = cpu_to_be16((u16)ppm->dgpu_tdp * 256 / 1000);
430 		dpm_table->PPM_TemperatureLimit = cpu_to_be16((u16)ppm->tj_max * 256);
431 	} else {
432 		dpm_table->PPM_PkgPwrLimit = cpu_to_be16(0);
433 		dpm_table->PPM_TemperatureLimit = cpu_to_be16(0);
434 	}
435 
436 	dpm_table->BAPM_TEMP_GRADIENT = cpu_to_be32(pt_defaults->bapm_temp_gradient);
437 	def1 = pt_defaults->bapmti_r;
438 	def2 = pt_defaults->bapmti_rc;
439 
440 	for (i = 0; i < SMU7_DTE_ITERATIONS; i++) {
441 		for (j = 0; j < SMU7_DTE_SOURCES; j++) {
442 			for (k = 0; k < SMU7_DTE_SINKS; k++) {
443 				dpm_table->BAPMTI_R[i][j][k] = cpu_to_be16(*def1);
444 				dpm_table->BAPMTI_RC[i][j][k] = cpu_to_be16(*def2);
445 				def1++;
446 				def2++;
447 			}
448 		}
449 	}
450 
451 	return 0;
452 }
453 
ci_populate_pm_base(struct radeon_device * rdev)454 static int ci_populate_pm_base(struct radeon_device *rdev)
455 {
456 	struct ci_power_info *pi = ci_get_pi(rdev);
457 	u32 pm_fuse_table_offset;
458 	int ret;
459 
460 	if (pi->caps_power_containment) {
461 		ret = ci_read_smc_sram_dword(rdev,
462 					     SMU7_FIRMWARE_HEADER_LOCATION +
463 					     offsetof(SMU7_Firmware_Header, PmFuseTable),
464 					     &pm_fuse_table_offset, pi->sram_end);
465 		if (ret)
466 			return ret;
467 		ret = ci_populate_bapm_vddc_vid_sidd(rdev);
468 		if (ret)
469 			return ret;
470 		ret = ci_populate_vddc_vid(rdev);
471 		if (ret)
472 			return ret;
473 		ret = ci_populate_svi_load_line(rdev);
474 		if (ret)
475 			return ret;
476 		ret = ci_populate_tdc_limit(rdev);
477 		if (ret)
478 			return ret;
479 		ret = ci_populate_dw8(rdev);
480 		if (ret)
481 			return ret;
482 		ret = ci_populate_fuzzy_fan(rdev);
483 		if (ret)
484 			return ret;
485 		ret = ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(rdev);
486 		if (ret)
487 			return ret;
488 		ret = ci_populate_bapm_vddc_base_leakage_sidd(rdev);
489 		if (ret)
490 			return ret;
491 		ret = ci_copy_bytes_to_smc(rdev, pm_fuse_table_offset,
492 					   (u8 *)&pi->smc_powertune_table,
493 					   sizeof(SMU7_Discrete_PmFuses), pi->sram_end);
494 		if (ret)
495 			return ret;
496 	}
497 
498 	return 0;
499 }
500 
ci_do_enable_didt(struct radeon_device * rdev,const bool enable)501 static void ci_do_enable_didt(struct radeon_device *rdev, const bool enable)
502 {
503 	struct ci_power_info *pi = ci_get_pi(rdev);
504 	u32 data;
505 
506 	if (pi->caps_sq_ramping) {
507 		data = RREG32_DIDT(DIDT_SQ_CTRL0);
508 		if (enable)
509 			data |= DIDT_CTRL_EN;
510 		else
511 			data &= ~DIDT_CTRL_EN;
512 		WREG32_DIDT(DIDT_SQ_CTRL0, data);
513 	}
514 
515 	if (pi->caps_db_ramping) {
516 		data = RREG32_DIDT(DIDT_DB_CTRL0);
517 		if (enable)
518 			data |= DIDT_CTRL_EN;
519 		else
520 			data &= ~DIDT_CTRL_EN;
521 		WREG32_DIDT(DIDT_DB_CTRL0, data);
522 	}
523 
524 	if (pi->caps_td_ramping) {
525 		data = RREG32_DIDT(DIDT_TD_CTRL0);
526 		if (enable)
527 			data |= DIDT_CTRL_EN;
528 		else
529 			data &= ~DIDT_CTRL_EN;
530 		WREG32_DIDT(DIDT_TD_CTRL0, data);
531 	}
532 
533 	if (pi->caps_tcp_ramping) {
534 		data = RREG32_DIDT(DIDT_TCP_CTRL0);
535 		if (enable)
536 			data |= DIDT_CTRL_EN;
537 		else
538 			data &= ~DIDT_CTRL_EN;
539 		WREG32_DIDT(DIDT_TCP_CTRL0, data);
540 	}
541 }
542 
ci_program_pt_config_registers(struct radeon_device * rdev,const struct ci_pt_config_reg * cac_config_regs)543 static int ci_program_pt_config_registers(struct radeon_device *rdev,
544 					  const struct ci_pt_config_reg *cac_config_regs)
545 {
546 	const struct ci_pt_config_reg *config_regs = cac_config_regs;
547 	u32 data;
548 	u32 cache = 0;
549 
550 	if (config_regs == NULL)
551 		return -EINVAL;
552 
553 	while (config_regs->offset != 0xFFFFFFFF) {
554 		if (config_regs->type == CISLANDS_CONFIGREG_CACHE) {
555 			cache |= ((config_regs->value << config_regs->shift) & config_regs->mask);
556 		} else {
557 			switch (config_regs->type) {
558 			case CISLANDS_CONFIGREG_SMC_IND:
559 				data = RREG32_SMC(config_regs->offset);
560 				break;
561 			case CISLANDS_CONFIGREG_DIDT_IND:
562 				data = RREG32_DIDT(config_regs->offset);
563 				break;
564 			default:
565 				data = RREG32(config_regs->offset << 2);
566 				break;
567 			}
568 
569 			data &= ~config_regs->mask;
570 			data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
571 			data |= cache;
572 
573 			switch (config_regs->type) {
574 			case CISLANDS_CONFIGREG_SMC_IND:
575 				WREG32_SMC(config_regs->offset, data);
576 				break;
577 			case CISLANDS_CONFIGREG_DIDT_IND:
578 				WREG32_DIDT(config_regs->offset, data);
579 				break;
580 			default:
581 				WREG32(config_regs->offset << 2, data);
582 				break;
583 			}
584 			cache = 0;
585 		}
586 		config_regs++;
587 	}
588 	return 0;
589 }
590 
ci_enable_didt(struct radeon_device * rdev,bool enable)591 static int ci_enable_didt(struct radeon_device *rdev, bool enable)
592 {
593 	struct ci_power_info *pi = ci_get_pi(rdev);
594 	int ret;
595 
596 	if (pi->caps_sq_ramping || pi->caps_db_ramping ||
597 	    pi->caps_td_ramping || pi->caps_tcp_ramping) {
598 		cik_enter_rlc_safe_mode(rdev);
599 
600 		if (enable) {
601 			ret = ci_program_pt_config_registers(rdev, didt_config_ci);
602 			if (ret) {
603 				cik_exit_rlc_safe_mode(rdev);
604 				return ret;
605 			}
606 		}
607 
608 		ci_do_enable_didt(rdev, enable);
609 
610 		cik_exit_rlc_safe_mode(rdev);
611 	}
612 
613 	return 0;
614 }
615 
ci_enable_power_containment(struct radeon_device * rdev,bool enable)616 static int ci_enable_power_containment(struct radeon_device *rdev, bool enable)
617 {
618 	struct ci_power_info *pi = ci_get_pi(rdev);
619 	PPSMC_Result smc_result;
620 	int ret = 0;
621 
622 	if (enable) {
623 		pi->power_containment_features = 0;
624 		if (pi->caps_power_containment) {
625 			if (pi->enable_bapm_feature) {
626 				smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableDTE);
627 				if (smc_result != PPSMC_Result_OK)
628 					ret = -EINVAL;
629 				else
630 					pi->power_containment_features |= POWERCONTAINMENT_FEATURE_BAPM;
631 			}
632 
633 			if (pi->enable_tdc_limit_feature) {
634 				smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_TDCLimitEnable);
635 				if (smc_result != PPSMC_Result_OK)
636 					ret = -EINVAL;
637 				else
638 					pi->power_containment_features |= POWERCONTAINMENT_FEATURE_TDCLimit;
639 			}
640 
641 			if (pi->enable_pkg_pwr_tracking_feature) {
642 				smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_PkgPwrLimitEnable);
643 				if (smc_result != PPSMC_Result_OK) {
644 					ret = -EINVAL;
645 				} else {
646 					struct radeon_cac_tdp_table *cac_tdp_table =
647 						rdev->pm.dpm.dyn_state.cac_tdp_table;
648 					u32 default_pwr_limit =
649 						(u32)(cac_tdp_table->maximum_power_delivery_limit * 256);
650 
651 					pi->power_containment_features |= POWERCONTAINMENT_FEATURE_PkgPwrLimit;
652 
653 					ci_set_power_limit(rdev, default_pwr_limit);
654 				}
655 			}
656 		}
657 	} else {
658 		if (pi->caps_power_containment && pi->power_containment_features) {
659 			if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_TDCLimit)
660 				ci_send_msg_to_smc(rdev, PPSMC_MSG_TDCLimitDisable);
661 
662 			if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_BAPM)
663 				ci_send_msg_to_smc(rdev, PPSMC_MSG_DisableDTE);
664 
665 			if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit)
666 				ci_send_msg_to_smc(rdev, PPSMC_MSG_PkgPwrLimitDisable);
667 			pi->power_containment_features = 0;
668 		}
669 	}
670 
671 	return ret;
672 }
673 
ci_enable_smc_cac(struct radeon_device * rdev,bool enable)674 static int ci_enable_smc_cac(struct radeon_device *rdev, bool enable)
675 {
676 	struct ci_power_info *pi = ci_get_pi(rdev);
677 	PPSMC_Result smc_result;
678 	int ret = 0;
679 
680 	if (pi->caps_cac) {
681 		if (enable) {
682 			smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
683 			if (smc_result != PPSMC_Result_OK) {
684 				ret = -EINVAL;
685 				pi->cac_enabled = false;
686 			} else {
687 				pi->cac_enabled = true;
688 			}
689 		} else if (pi->cac_enabled) {
690 			ci_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
691 			pi->cac_enabled = false;
692 		}
693 	}
694 
695 	return ret;
696 }
697 
ci_enable_thermal_based_sclk_dpm(struct radeon_device * rdev,bool enable)698 static int ci_enable_thermal_based_sclk_dpm(struct radeon_device *rdev,
699 					    bool enable)
700 {
701 	struct ci_power_info *pi = ci_get_pi(rdev);
702 	PPSMC_Result smc_result = PPSMC_Result_OK;
703 
704 	if (pi->thermal_sclk_dpm_enabled) {
705 		if (enable)
706 			smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_ENABLE_THERMAL_DPM);
707 		else
708 			smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DISABLE_THERMAL_DPM);
709 	}
710 
711 	if (smc_result == PPSMC_Result_OK)
712 		return 0;
713 	else
714 		return -EINVAL;
715 }
716 
ci_power_control_set_level(struct radeon_device * rdev)717 static int ci_power_control_set_level(struct radeon_device *rdev)
718 {
719 	struct ci_power_info *pi = ci_get_pi(rdev);
720 	struct radeon_cac_tdp_table *cac_tdp_table =
721 		rdev->pm.dpm.dyn_state.cac_tdp_table;
722 	s32 adjust_percent;
723 	s32 target_tdp;
724 	int ret = 0;
725 	bool adjust_polarity = false; /* ??? */
726 
727 	if (pi->caps_power_containment) {
728 		adjust_percent = adjust_polarity ?
729 			rdev->pm.dpm.tdp_adjustment : (-1 * rdev->pm.dpm.tdp_adjustment);
730 		target_tdp = ((100 + adjust_percent) *
731 			      (s32)cac_tdp_table->configurable_tdp) / 100;
732 
733 		ret = ci_set_overdrive_target_tdp(rdev, (u32)target_tdp);
734 	}
735 
736 	return ret;
737 }
738 
ci_dpm_powergate_uvd(struct radeon_device * rdev,bool gate)739 void ci_dpm_powergate_uvd(struct radeon_device *rdev, bool gate)
740 {
741 	struct ci_power_info *pi = ci_get_pi(rdev);
742 
743 	if (pi->uvd_power_gated == gate)
744 		return;
745 
746 	pi->uvd_power_gated = gate;
747 
748 	ci_update_uvd_dpm(rdev, gate);
749 }
750 
ci_dpm_vblank_too_short(struct radeon_device * rdev)751 bool ci_dpm_vblank_too_short(struct radeon_device *rdev)
752 {
753 	struct ci_power_info *pi = ci_get_pi(rdev);
754 	u32 vblank_time = r600_dpm_get_vblank_time(rdev);
755 	u32 switch_limit = pi->mem_gddr5 ? 450 : 300;
756 
757 	/* disable mclk switching if the refresh is >120Hz, even if the
758         * blanking period would allow it
759         */
760 	if (r600_dpm_get_vrefresh(rdev) > 120)
761 		return true;
762 
763 	if (vblank_time < switch_limit)
764 		return true;
765 	else
766 		return false;
767 
768 }
769 
ci_apply_state_adjust_rules(struct radeon_device * rdev,struct radeon_ps * rps)770 static void ci_apply_state_adjust_rules(struct radeon_device *rdev,
771 					struct radeon_ps *rps)
772 {
773 	struct ci_ps *ps = ci_get_ps(rps);
774 	struct ci_power_info *pi = ci_get_pi(rdev);
775 	struct radeon_clock_and_voltage_limits *max_limits;
776 	bool disable_mclk_switching;
777 	u32 sclk, mclk;
778 	int i;
779 
780 	if (rps->vce_active) {
781 		rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
782 		rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
783 	} else {
784 		rps->evclk = 0;
785 		rps->ecclk = 0;
786 	}
787 
788 	if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
789 	    ci_dpm_vblank_too_short(rdev))
790 		disable_mclk_switching = true;
791 	else
792 		disable_mclk_switching = false;
793 
794 	if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)
795 		pi->battery_state = true;
796 	else
797 		pi->battery_state = false;
798 
799 	if (rdev->pm.dpm.ac_power)
800 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
801 	else
802 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
803 
804 	if (rdev->pm.dpm.ac_power == false) {
805 		for (i = 0; i < ps->performance_level_count; i++) {
806 			if (ps->performance_levels[i].mclk > max_limits->mclk)
807 				ps->performance_levels[i].mclk = max_limits->mclk;
808 			if (ps->performance_levels[i].sclk > max_limits->sclk)
809 				ps->performance_levels[i].sclk = max_limits->sclk;
810 		}
811 	}
812 
813 	/* XXX validate the min clocks required for display */
814 
815 	if (disable_mclk_switching) {
816 		mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
817 		sclk = ps->performance_levels[0].sclk;
818 	} else {
819 		mclk = ps->performance_levels[0].mclk;
820 		sclk = ps->performance_levels[0].sclk;
821 	}
822 
823 	if (rps->vce_active) {
824 		if (sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
825 			sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
826 		if (mclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk)
827 			mclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk;
828 	}
829 
830 	ps->performance_levels[0].sclk = sclk;
831 	ps->performance_levels[0].mclk = mclk;
832 
833 	if (ps->performance_levels[1].sclk < ps->performance_levels[0].sclk)
834 		ps->performance_levels[1].sclk = ps->performance_levels[0].sclk;
835 
836 	if (disable_mclk_switching) {
837 		if (ps->performance_levels[0].mclk < ps->performance_levels[1].mclk)
838 			ps->performance_levels[0].mclk = ps->performance_levels[1].mclk;
839 	} else {
840 		if (ps->performance_levels[1].mclk < ps->performance_levels[0].mclk)
841 			ps->performance_levels[1].mclk = ps->performance_levels[0].mclk;
842 	}
843 }
844 
ci_thermal_set_temperature_range(struct radeon_device * rdev,int min_temp,int max_temp)845 static int ci_thermal_set_temperature_range(struct radeon_device *rdev,
846 					    int min_temp, int max_temp)
847 {
848 	int low_temp = 0 * 1000;
849 	int high_temp = 255 * 1000;
850 	u32 tmp;
851 
852 	if (low_temp < min_temp)
853 		low_temp = min_temp;
854 	if (high_temp > max_temp)
855 		high_temp = max_temp;
856 	if (high_temp < low_temp) {
857 		DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
858 		return -EINVAL;
859 	}
860 
861 	tmp = RREG32_SMC(CG_THERMAL_INT);
862 	tmp &= ~(CI_DIG_THERM_INTH_MASK | CI_DIG_THERM_INTL_MASK);
863 	tmp |= CI_DIG_THERM_INTH(high_temp / 1000) |
864 		CI_DIG_THERM_INTL(low_temp / 1000);
865 	WREG32_SMC(CG_THERMAL_INT, tmp);
866 
867 #if 0
868 	/* XXX: need to figure out how to handle this properly */
869 	tmp = RREG32_SMC(CG_THERMAL_CTRL);
870 	tmp &= DIG_THERM_DPM_MASK;
871 	tmp |= DIG_THERM_DPM(high_temp / 1000);
872 	WREG32_SMC(CG_THERMAL_CTRL, tmp);
873 #endif
874 
875 	rdev->pm.dpm.thermal.min_temp = low_temp;
876 	rdev->pm.dpm.thermal.max_temp = high_temp;
877 
878 	return 0;
879 }
880 
ci_thermal_enable_alert(struct radeon_device * rdev,bool enable)881 static int ci_thermal_enable_alert(struct radeon_device *rdev,
882 				   bool enable)
883 {
884 	u32 thermal_int = RREG32_SMC(CG_THERMAL_INT);
885 	PPSMC_Result result;
886 
887 	if (enable) {
888 		thermal_int &= ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
889 		WREG32_SMC(CG_THERMAL_INT, thermal_int);
890 		rdev->irq.dpm_thermal = false;
891 		result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Thermal_Cntl_Enable);
892 		if (result != PPSMC_Result_OK) {
893 			DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
894 			return -EINVAL;
895 		}
896 	} else {
897 		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
898 		WREG32_SMC(CG_THERMAL_INT, thermal_int);
899 		rdev->irq.dpm_thermal = true;
900 		result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Thermal_Cntl_Disable);
901 		if (result != PPSMC_Result_OK) {
902 			DRM_DEBUG_KMS("Could not disable thermal interrupts.\n");
903 			return -EINVAL;
904 		}
905 	}
906 
907 	return 0;
908 }
909 
ci_fan_ctrl_set_static_mode(struct radeon_device * rdev,u32 mode)910 static void ci_fan_ctrl_set_static_mode(struct radeon_device *rdev, u32 mode)
911 {
912 	struct ci_power_info *pi = ci_get_pi(rdev);
913 	u32 tmp;
914 
915 	if (pi->fan_ctrl_is_in_default_mode) {
916 		tmp = (RREG32_SMC(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK) >> FDO_PWM_MODE_SHIFT;
917 		pi->fan_ctrl_default_mode = tmp;
918 		tmp = (RREG32_SMC(CG_FDO_CTRL2) & TMIN_MASK) >> TMIN_SHIFT;
919 		pi->t_min = tmp;
920 		pi->fan_ctrl_is_in_default_mode = false;
921 	}
922 
923 	tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TMIN_MASK;
924 	tmp |= TMIN(0);
925 	WREG32_SMC(CG_FDO_CTRL2, tmp);
926 
927 	tmp = RREG32_SMC(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK;
928 	tmp |= FDO_PWM_MODE(mode);
929 	WREG32_SMC(CG_FDO_CTRL2, tmp);
930 }
931 
ci_thermal_setup_fan_table(struct radeon_device * rdev)932 static int ci_thermal_setup_fan_table(struct radeon_device *rdev)
933 {
934 	struct ci_power_info *pi = ci_get_pi(rdev);
935 	SMU7_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
936 	u32 duty100;
937 	u32 t_diff1, t_diff2, pwm_diff1, pwm_diff2;
938 	u16 fdo_min, slope1, slope2;
939 	u32 reference_clock, tmp;
940 	int ret;
941 	u64 tmp64;
942 
943 	if (!pi->fan_table_start) {
944 		rdev->pm.dpm.fan.ucode_fan_control = false;
945 		return 0;
946 	}
947 
948 	duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT;
949 
950 	if (duty100 == 0) {
951 		rdev->pm.dpm.fan.ucode_fan_control = false;
952 		return 0;
953 	}
954 
955 	tmp64 = (u64)rdev->pm.dpm.fan.pwm_min * duty100;
956 	do_div(tmp64, 10000);
957 	fdo_min = (u16)tmp64;
958 
959 	t_diff1 = rdev->pm.dpm.fan.t_med - rdev->pm.dpm.fan.t_min;
960 	t_diff2 = rdev->pm.dpm.fan.t_high - rdev->pm.dpm.fan.t_med;
961 
962 	pwm_diff1 = rdev->pm.dpm.fan.pwm_med - rdev->pm.dpm.fan.pwm_min;
963 	pwm_diff2 = rdev->pm.dpm.fan.pwm_high - rdev->pm.dpm.fan.pwm_med;
964 
965 	slope1 = (u16)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
966 	slope2 = (u16)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
967 
968 	fan_table.TempMin = cpu_to_be16((50 + rdev->pm.dpm.fan.t_min) / 100);
969 	fan_table.TempMed = cpu_to_be16((50 + rdev->pm.dpm.fan.t_med) / 100);
970 	fan_table.TempMax = cpu_to_be16((50 + rdev->pm.dpm.fan.t_max) / 100);
971 
972 	fan_table.Slope1 = cpu_to_be16(slope1);
973 	fan_table.Slope2 = cpu_to_be16(slope2);
974 
975 	fan_table.FdoMin = cpu_to_be16(fdo_min);
976 
977 	fan_table.HystDown = cpu_to_be16(rdev->pm.dpm.fan.t_hyst);
978 
979 	fan_table.HystUp = cpu_to_be16(1);
980 
981 	fan_table.HystSlope = cpu_to_be16(1);
982 
983 	fan_table.TempRespLim = cpu_to_be16(5);
984 
985 	reference_clock = radeon_get_xclk(rdev);
986 
987 	fan_table.RefreshPeriod = cpu_to_be32((rdev->pm.dpm.fan.cycle_delay *
988 					       reference_clock) / 1600);
989 
990 	fan_table.FdoMax = cpu_to_be16((u16)duty100);
991 
992 	tmp = (RREG32_SMC(CG_MULT_THERMAL_CTRL) & TEMP_SEL_MASK) >> TEMP_SEL_SHIFT;
993 	fan_table.TempSrc = (uint8_t)tmp;
994 
995 	ret = ci_copy_bytes_to_smc(rdev,
996 				   pi->fan_table_start,
997 				   (u8 *)(&fan_table),
998 				   sizeof(fan_table),
999 				   pi->sram_end);
1000 
1001 	if (ret) {
1002 		DRM_ERROR("Failed to load fan table to the SMC.");
1003 		rdev->pm.dpm.fan.ucode_fan_control = false;
1004 	}
1005 
1006 	return 0;
1007 }
1008 
ci_fan_ctrl_start_smc_fan_control(struct radeon_device * rdev)1009 static int ci_fan_ctrl_start_smc_fan_control(struct radeon_device *rdev)
1010 {
1011 	struct ci_power_info *pi = ci_get_pi(rdev);
1012 	PPSMC_Result ret;
1013 
1014 	if (pi->caps_od_fuzzy_fan_control_support) {
1015 		ret = ci_send_msg_to_smc_with_parameter(rdev,
1016 							PPSMC_StartFanControl,
1017 							FAN_CONTROL_FUZZY);
1018 		if (ret != PPSMC_Result_OK)
1019 			return -EINVAL;
1020 		ret = ci_send_msg_to_smc_with_parameter(rdev,
1021 							PPSMC_MSG_SetFanPwmMax,
1022 							rdev->pm.dpm.fan.default_max_fan_pwm);
1023 		if (ret != PPSMC_Result_OK)
1024 			return -EINVAL;
1025 	} else {
1026 		ret = ci_send_msg_to_smc_with_parameter(rdev,
1027 							PPSMC_StartFanControl,
1028 							FAN_CONTROL_TABLE);
1029 		if (ret != PPSMC_Result_OK)
1030 			return -EINVAL;
1031 	}
1032 
1033 	pi->fan_is_controlled_by_smc = true;
1034 	return 0;
1035 }
1036 
ci_fan_ctrl_stop_smc_fan_control(struct radeon_device * rdev)1037 static int ci_fan_ctrl_stop_smc_fan_control(struct radeon_device *rdev)
1038 {
1039 	PPSMC_Result ret;
1040 	struct ci_power_info *pi = ci_get_pi(rdev);
1041 
1042 	ret = ci_send_msg_to_smc(rdev, PPSMC_StopFanControl);
1043 	if (ret == PPSMC_Result_OK) {
1044 		pi->fan_is_controlled_by_smc = false;
1045 		return 0;
1046 	} else
1047 		return -EINVAL;
1048 }
1049 
ci_fan_ctrl_get_fan_speed_percent(struct radeon_device * rdev,u32 * speed)1050 int ci_fan_ctrl_get_fan_speed_percent(struct radeon_device *rdev,
1051 					     u32 *speed)
1052 {
1053 	u32 duty, duty100;
1054 	u64 tmp64;
1055 
1056 	if (rdev->pm.no_fan)
1057 		return -ENOENT;
1058 
1059 	duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT;
1060 	duty = (RREG32_SMC(CG_THERMAL_STATUS) & FDO_PWM_DUTY_MASK) >> FDO_PWM_DUTY_SHIFT;
1061 
1062 	if (duty100 == 0)
1063 		return -EINVAL;
1064 
1065 	tmp64 = (u64)duty * 100;
1066 	do_div(tmp64, duty100);
1067 	*speed = (u32)tmp64;
1068 
1069 	if (*speed > 100)
1070 		*speed = 100;
1071 
1072 	return 0;
1073 }
1074 
ci_fan_ctrl_set_fan_speed_percent(struct radeon_device * rdev,u32 speed)1075 int ci_fan_ctrl_set_fan_speed_percent(struct radeon_device *rdev,
1076 					     u32 speed)
1077 {
1078 	u32 tmp;
1079 	u32 duty, duty100;
1080 	u64 tmp64;
1081 	struct ci_power_info *pi = ci_get_pi(rdev);
1082 
1083 	if (rdev->pm.no_fan)
1084 		return -ENOENT;
1085 
1086 	if (pi->fan_is_controlled_by_smc)
1087 		return -EINVAL;
1088 
1089 	if (speed > 100)
1090 		return -EINVAL;
1091 
1092 	duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT;
1093 
1094 	if (duty100 == 0)
1095 		return -EINVAL;
1096 
1097 	tmp64 = (u64)speed * duty100;
1098 	do_div(tmp64, 100);
1099 	duty = (u32)tmp64;
1100 
1101 	tmp = RREG32_SMC(CG_FDO_CTRL0) & ~FDO_STATIC_DUTY_MASK;
1102 	tmp |= FDO_STATIC_DUTY(duty);
1103 	WREG32_SMC(CG_FDO_CTRL0, tmp);
1104 
1105 	return 0;
1106 }
1107 
ci_fan_ctrl_set_mode(struct radeon_device * rdev,u32 mode)1108 void ci_fan_ctrl_set_mode(struct radeon_device *rdev, u32 mode)
1109 {
1110 	if (mode) {
1111 		/* stop auto-manage */
1112 		if (rdev->pm.dpm.fan.ucode_fan_control)
1113 			ci_fan_ctrl_stop_smc_fan_control(rdev);
1114 		ci_fan_ctrl_set_static_mode(rdev, mode);
1115 	} else {
1116 		/* restart auto-manage */
1117 		if (rdev->pm.dpm.fan.ucode_fan_control)
1118 			ci_thermal_start_smc_fan_control(rdev);
1119 		else
1120 			ci_fan_ctrl_set_default_mode(rdev);
1121 	}
1122 }
1123 
ci_fan_ctrl_get_mode(struct radeon_device * rdev)1124 u32 ci_fan_ctrl_get_mode(struct radeon_device *rdev)
1125 {
1126 	struct ci_power_info *pi = ci_get_pi(rdev);
1127 	u32 tmp;
1128 
1129 	if (pi->fan_is_controlled_by_smc)
1130 		return 0;
1131 
1132 	tmp = RREG32_SMC(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK;
1133 	return (tmp >> FDO_PWM_MODE_SHIFT);
1134 }
1135 
1136 #if 0
1137 static int ci_fan_ctrl_get_fan_speed_rpm(struct radeon_device *rdev,
1138 					 u32 *speed)
1139 {
1140 	u32 tach_period;
1141 	u32 xclk = radeon_get_xclk(rdev);
1142 
1143 	if (rdev->pm.no_fan)
1144 		return -ENOENT;
1145 
1146 	if (rdev->pm.fan_pulses_per_revolution == 0)
1147 		return -ENOENT;
1148 
1149 	tach_period = (RREG32_SMC(CG_TACH_STATUS) & TACH_PERIOD_MASK) >> TACH_PERIOD_SHIFT;
1150 	if (tach_period == 0)
1151 		return -ENOENT;
1152 
1153 	*speed = 60 * xclk * 10000 / tach_period;
1154 
1155 	return 0;
1156 }
1157 
1158 static int ci_fan_ctrl_set_fan_speed_rpm(struct radeon_device *rdev,
1159 					 u32 speed)
1160 {
1161 	u32 tach_period, tmp;
1162 	u32 xclk = radeon_get_xclk(rdev);
1163 
1164 	if (rdev->pm.no_fan)
1165 		return -ENOENT;
1166 
1167 	if (rdev->pm.fan_pulses_per_revolution == 0)
1168 		return -ENOENT;
1169 
1170 	if ((speed < rdev->pm.fan_min_rpm) ||
1171 	    (speed > rdev->pm.fan_max_rpm))
1172 		return -EINVAL;
1173 
1174 	if (rdev->pm.dpm.fan.ucode_fan_control)
1175 		ci_fan_ctrl_stop_smc_fan_control(rdev);
1176 
1177 	tach_period = 60 * xclk * 10000 / (8 * speed);
1178 	tmp = RREG32_SMC(CG_TACH_CTRL) & ~TARGET_PERIOD_MASK;
1179 	tmp |= TARGET_PERIOD(tach_period);
1180 	WREG32_SMC(CG_TACH_CTRL, tmp);
1181 
1182 	ci_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC_RPM);
1183 
1184 	return 0;
1185 }
1186 #endif
1187 
ci_fan_ctrl_set_default_mode(struct radeon_device * rdev)1188 static void ci_fan_ctrl_set_default_mode(struct radeon_device *rdev)
1189 {
1190 	struct ci_power_info *pi = ci_get_pi(rdev);
1191 	u32 tmp;
1192 
1193 	if (!pi->fan_ctrl_is_in_default_mode) {
1194 		tmp = RREG32_SMC(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK;
1195 		tmp |= FDO_PWM_MODE(pi->fan_ctrl_default_mode);
1196 		WREG32_SMC(CG_FDO_CTRL2, tmp);
1197 
1198 		tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TMIN_MASK;
1199 		tmp |= TMIN(pi->t_min);
1200 		WREG32_SMC(CG_FDO_CTRL2, tmp);
1201 		pi->fan_ctrl_is_in_default_mode = true;
1202 	}
1203 }
1204 
ci_thermal_start_smc_fan_control(struct radeon_device * rdev)1205 static void ci_thermal_start_smc_fan_control(struct radeon_device *rdev)
1206 {
1207 	if (rdev->pm.dpm.fan.ucode_fan_control) {
1208 		ci_fan_ctrl_start_smc_fan_control(rdev);
1209 		ci_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC);
1210 	}
1211 }
1212 
ci_thermal_initialize(struct radeon_device * rdev)1213 static void ci_thermal_initialize(struct radeon_device *rdev)
1214 {
1215 	u32 tmp;
1216 
1217 	if (rdev->pm.fan_pulses_per_revolution) {
1218 		tmp = RREG32_SMC(CG_TACH_CTRL) & ~EDGE_PER_REV_MASK;
1219 		tmp |= EDGE_PER_REV(rdev->pm.fan_pulses_per_revolution -1);
1220 		WREG32_SMC(CG_TACH_CTRL, tmp);
1221 	}
1222 
1223 	tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TACH_PWM_RESP_RATE_MASK;
1224 	tmp |= TACH_PWM_RESP_RATE(0x28);
1225 	WREG32_SMC(CG_FDO_CTRL2, tmp);
1226 }
1227 
ci_thermal_start_thermal_controller(struct radeon_device * rdev)1228 static int ci_thermal_start_thermal_controller(struct radeon_device *rdev)
1229 {
1230 	int ret;
1231 
1232 	ci_thermal_initialize(rdev);
1233 	ret = ci_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1234 	if (ret)
1235 		return ret;
1236 	ret = ci_thermal_enable_alert(rdev, true);
1237 	if (ret)
1238 		return ret;
1239 	if (rdev->pm.dpm.fan.ucode_fan_control) {
1240 		ret = ci_thermal_setup_fan_table(rdev);
1241 		if (ret)
1242 			return ret;
1243 		ci_thermal_start_smc_fan_control(rdev);
1244 	}
1245 
1246 	return 0;
1247 }
1248 
ci_thermal_stop_thermal_controller(struct radeon_device * rdev)1249 static void ci_thermal_stop_thermal_controller(struct radeon_device *rdev)
1250 {
1251 	if (!rdev->pm.no_fan)
1252 		ci_fan_ctrl_set_default_mode(rdev);
1253 }
1254 
1255 #if 0
1256 static int ci_read_smc_soft_register(struct radeon_device *rdev,
1257 				     u16 reg_offset, u32 *value)
1258 {
1259 	struct ci_power_info *pi = ci_get_pi(rdev);
1260 
1261 	return ci_read_smc_sram_dword(rdev,
1262 				      pi->soft_regs_start + reg_offset,
1263 				      value, pi->sram_end);
1264 }
1265 #endif
1266 
ci_write_smc_soft_register(struct radeon_device * rdev,u16 reg_offset,u32 value)1267 static int ci_write_smc_soft_register(struct radeon_device *rdev,
1268 				      u16 reg_offset, u32 value)
1269 {
1270 	struct ci_power_info *pi = ci_get_pi(rdev);
1271 
1272 	return ci_write_smc_sram_dword(rdev,
1273 				       pi->soft_regs_start + reg_offset,
1274 				       value, pi->sram_end);
1275 }
1276 
ci_init_fps_limits(struct radeon_device * rdev)1277 static void ci_init_fps_limits(struct radeon_device *rdev)
1278 {
1279 	struct ci_power_info *pi = ci_get_pi(rdev);
1280 	SMU7_Discrete_DpmTable *table = &pi->smc_state_table;
1281 
1282 	if (pi->caps_fps) {
1283 		u16 tmp;
1284 
1285 		tmp = 45;
1286 		table->FpsHighT = cpu_to_be16(tmp);
1287 
1288 		tmp = 30;
1289 		table->FpsLowT = cpu_to_be16(tmp);
1290 	}
1291 }
1292 
ci_update_sclk_t(struct radeon_device * rdev)1293 static int ci_update_sclk_t(struct radeon_device *rdev)
1294 {
1295 	struct ci_power_info *pi = ci_get_pi(rdev);
1296 	int ret = 0;
1297 	u32 low_sclk_interrupt_t = 0;
1298 
1299 	if (pi->caps_sclk_throttle_low_notification) {
1300 		low_sclk_interrupt_t = cpu_to_be32(pi->low_sclk_interrupt_t);
1301 
1302 		ret = ci_copy_bytes_to_smc(rdev,
1303 					   pi->dpm_table_start +
1304 					   offsetof(SMU7_Discrete_DpmTable, LowSclkInterruptT),
1305 					   (u8 *)&low_sclk_interrupt_t,
1306 					   sizeof(u32), pi->sram_end);
1307 
1308 	}
1309 
1310 	return ret;
1311 }
1312 
ci_get_leakage_voltages(struct radeon_device * rdev)1313 static void ci_get_leakage_voltages(struct radeon_device *rdev)
1314 {
1315 	struct ci_power_info *pi = ci_get_pi(rdev);
1316 	u16 leakage_id, virtual_voltage_id;
1317 	u16 vddc, vddci;
1318 	int i;
1319 
1320 	pi->vddc_leakage.count = 0;
1321 	pi->vddci_leakage.count = 0;
1322 
1323 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) {
1324 		for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) {
1325 			virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
1326 			if (radeon_atom_get_voltage_evv(rdev, virtual_voltage_id, &vddc) != 0)
1327 				continue;
1328 			if (vddc != 0 && vddc != virtual_voltage_id) {
1329 				pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc;
1330 				pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id;
1331 				pi->vddc_leakage.count++;
1332 			}
1333 		}
1334 	} else if (radeon_atom_get_leakage_id_from_vbios(rdev, &leakage_id) == 0) {
1335 		for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) {
1336 			virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
1337 			if (radeon_atom_get_leakage_vddc_based_on_leakage_params(rdev, &vddc, &vddci,
1338 										 virtual_voltage_id,
1339 										 leakage_id) == 0) {
1340 				if (vddc != 0 && vddc != virtual_voltage_id) {
1341 					pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc;
1342 					pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id;
1343 					pi->vddc_leakage.count++;
1344 				}
1345 				if (vddci != 0 && vddci != virtual_voltage_id) {
1346 					pi->vddci_leakage.actual_voltage[pi->vddci_leakage.count] = vddci;
1347 					pi->vddci_leakage.leakage_id[pi->vddci_leakage.count] = virtual_voltage_id;
1348 					pi->vddci_leakage.count++;
1349 				}
1350 			}
1351 		}
1352 	}
1353 }
1354 
ci_set_dpm_event_sources(struct radeon_device * rdev,u32 sources)1355 static void ci_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
1356 {
1357 	struct ci_power_info *pi = ci_get_pi(rdev);
1358 	bool want_thermal_protection;
1359 	u32 tmp;
1360 
1361 	switch (sources) {
1362 	case 0:
1363 	default:
1364 		want_thermal_protection = false;
1365 		break;
1366 	case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
1367 		want_thermal_protection = true;
1368 		break;
1369 	case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
1370 		want_thermal_protection = true;
1371 		break;
1372 	case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
1373 	      (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
1374 		want_thermal_protection = true;
1375 		break;
1376 	}
1377 
1378 	if (want_thermal_protection) {
1379 		tmp = RREG32_SMC(GENERAL_PWRMGT);
1380 		if (pi->thermal_protection)
1381 			tmp &= ~THERMAL_PROTECTION_DIS;
1382 		else
1383 			tmp |= THERMAL_PROTECTION_DIS;
1384 		WREG32_SMC(GENERAL_PWRMGT, tmp);
1385 	} else {
1386 		tmp = RREG32_SMC(GENERAL_PWRMGT);
1387 		tmp |= THERMAL_PROTECTION_DIS;
1388 		WREG32_SMC(GENERAL_PWRMGT, tmp);
1389 	}
1390 }
1391 
ci_enable_auto_throttle_source(struct radeon_device * rdev,enum radeon_dpm_auto_throttle_src source,bool enable)1392 static void ci_enable_auto_throttle_source(struct radeon_device *rdev,
1393 					   enum radeon_dpm_auto_throttle_src source,
1394 					   bool enable)
1395 {
1396 	struct ci_power_info *pi = ci_get_pi(rdev);
1397 
1398 	if (enable) {
1399 		if (!(pi->active_auto_throttle_sources & (1 << source))) {
1400 			pi->active_auto_throttle_sources |= 1 << source;
1401 			ci_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1402 		}
1403 	} else {
1404 		if (pi->active_auto_throttle_sources & (1 << source)) {
1405 			pi->active_auto_throttle_sources &= ~(1 << source);
1406 			ci_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1407 		}
1408 	}
1409 }
1410 
ci_enable_vr_hot_gpio_interrupt(struct radeon_device * rdev)1411 static void ci_enable_vr_hot_gpio_interrupt(struct radeon_device *rdev)
1412 {
1413 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1414 		ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableVRHotGPIOInterrupt);
1415 }
1416 
ci_unfreeze_sclk_mclk_dpm(struct radeon_device * rdev)1417 static int ci_unfreeze_sclk_mclk_dpm(struct radeon_device *rdev)
1418 {
1419 	struct ci_power_info *pi = ci_get_pi(rdev);
1420 	PPSMC_Result smc_result;
1421 
1422 	if (!pi->need_update_smu7_dpm_table)
1423 		return 0;
1424 
1425 	if ((!pi->sclk_dpm_key_disabled) &&
1426 	    (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) {
1427 		smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_SCLKDPM_UnfreezeLevel);
1428 		if (smc_result != PPSMC_Result_OK)
1429 			return -EINVAL;
1430 	}
1431 
1432 	if ((!pi->mclk_dpm_key_disabled) &&
1433 	    (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
1434 		smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_UnfreezeLevel);
1435 		if (smc_result != PPSMC_Result_OK)
1436 			return -EINVAL;
1437 	}
1438 
1439 	pi->need_update_smu7_dpm_table = 0;
1440 	return 0;
1441 }
1442 
ci_enable_sclk_mclk_dpm(struct radeon_device * rdev,bool enable)1443 static int ci_enable_sclk_mclk_dpm(struct radeon_device *rdev, bool enable)
1444 {
1445 	struct ci_power_info *pi = ci_get_pi(rdev);
1446 	PPSMC_Result smc_result;
1447 
1448 	if (enable) {
1449 		if (!pi->sclk_dpm_key_disabled) {
1450 			smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DPM_Enable);
1451 			if (smc_result != PPSMC_Result_OK)
1452 				return -EINVAL;
1453 		}
1454 
1455 		if (!pi->mclk_dpm_key_disabled) {
1456 			smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_Enable);
1457 			if (smc_result != PPSMC_Result_OK)
1458 				return -EINVAL;
1459 
1460 			WREG32_P(MC_SEQ_CNTL_3, CAC_EN, ~CAC_EN);
1461 
1462 			WREG32_SMC(LCAC_MC0_CNTL, 0x05);
1463 			WREG32_SMC(LCAC_MC1_CNTL, 0x05);
1464 			WREG32_SMC(LCAC_CPL_CNTL, 0x100005);
1465 
1466 			udelay(10);
1467 
1468 			WREG32_SMC(LCAC_MC0_CNTL, 0x400005);
1469 			WREG32_SMC(LCAC_MC1_CNTL, 0x400005);
1470 			WREG32_SMC(LCAC_CPL_CNTL, 0x500005);
1471 		}
1472 	} else {
1473 		if (!pi->sclk_dpm_key_disabled) {
1474 			smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DPM_Disable);
1475 			if (smc_result != PPSMC_Result_OK)
1476 				return -EINVAL;
1477 		}
1478 
1479 		if (!pi->mclk_dpm_key_disabled) {
1480 			smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_Disable);
1481 			if (smc_result != PPSMC_Result_OK)
1482 				return -EINVAL;
1483 		}
1484 	}
1485 
1486 	return 0;
1487 }
1488 
ci_start_dpm(struct radeon_device * rdev)1489 static int ci_start_dpm(struct radeon_device *rdev)
1490 {
1491 	struct ci_power_info *pi = ci_get_pi(rdev);
1492 	PPSMC_Result smc_result;
1493 	int ret;
1494 	u32 tmp;
1495 
1496 	tmp = RREG32_SMC(GENERAL_PWRMGT);
1497 	tmp |= GLOBAL_PWRMGT_EN;
1498 	WREG32_SMC(GENERAL_PWRMGT, tmp);
1499 
1500 	tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
1501 	tmp |= DYNAMIC_PM_EN;
1502 	WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
1503 
1504 	ci_write_smc_soft_register(rdev, offsetof(SMU7_SoftRegisters, VoltageChangeTimeout), 0x1000);
1505 
1506 	WREG32_P(BIF_LNCNT_RESET, 0, ~RESET_LNCNT_EN);
1507 
1508 	smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Voltage_Cntl_Enable);
1509 	if (smc_result != PPSMC_Result_OK)
1510 		return -EINVAL;
1511 
1512 	ret = ci_enable_sclk_mclk_dpm(rdev, true);
1513 	if (ret)
1514 		return ret;
1515 
1516 	if (!pi->pcie_dpm_key_disabled) {
1517 		smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_PCIeDPM_Enable);
1518 		if (smc_result != PPSMC_Result_OK)
1519 			return -EINVAL;
1520 	}
1521 
1522 	return 0;
1523 }
1524 
ci_freeze_sclk_mclk_dpm(struct radeon_device * rdev)1525 static int ci_freeze_sclk_mclk_dpm(struct radeon_device *rdev)
1526 {
1527 	struct ci_power_info *pi = ci_get_pi(rdev);
1528 	PPSMC_Result smc_result;
1529 
1530 	if (!pi->need_update_smu7_dpm_table)
1531 		return 0;
1532 
1533 	if ((!pi->sclk_dpm_key_disabled) &&
1534 	    (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) {
1535 		smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_SCLKDPM_FreezeLevel);
1536 		if (smc_result != PPSMC_Result_OK)
1537 			return -EINVAL;
1538 	}
1539 
1540 	if ((!pi->mclk_dpm_key_disabled) &&
1541 	    (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
1542 		smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_FreezeLevel);
1543 		if (smc_result != PPSMC_Result_OK)
1544 			return -EINVAL;
1545 	}
1546 
1547 	return 0;
1548 }
1549 
ci_stop_dpm(struct radeon_device * rdev)1550 static int ci_stop_dpm(struct radeon_device *rdev)
1551 {
1552 	struct ci_power_info *pi = ci_get_pi(rdev);
1553 	PPSMC_Result smc_result;
1554 	int ret;
1555 	u32 tmp;
1556 
1557 	tmp = RREG32_SMC(GENERAL_PWRMGT);
1558 	tmp &= ~GLOBAL_PWRMGT_EN;
1559 	WREG32_SMC(GENERAL_PWRMGT, tmp);
1560 
1561 	tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
1562 	tmp &= ~DYNAMIC_PM_EN;
1563 	WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
1564 
1565 	if (!pi->pcie_dpm_key_disabled) {
1566 		smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_PCIeDPM_Disable);
1567 		if (smc_result != PPSMC_Result_OK)
1568 			return -EINVAL;
1569 	}
1570 
1571 	ret = ci_enable_sclk_mclk_dpm(rdev, false);
1572 	if (ret)
1573 		return ret;
1574 
1575 	smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Voltage_Cntl_Disable);
1576 	if (smc_result != PPSMC_Result_OK)
1577 		return -EINVAL;
1578 
1579 	return 0;
1580 }
1581 
ci_enable_sclk_control(struct radeon_device * rdev,bool enable)1582 static void ci_enable_sclk_control(struct radeon_device *rdev, bool enable)
1583 {
1584 	u32 tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
1585 
1586 	if (enable)
1587 		tmp &= ~SCLK_PWRMGT_OFF;
1588 	else
1589 		tmp |= SCLK_PWRMGT_OFF;
1590 	WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
1591 }
1592 
1593 #if 0
1594 static int ci_notify_hw_of_power_source(struct radeon_device *rdev,
1595 					bool ac_power)
1596 {
1597 	struct ci_power_info *pi = ci_get_pi(rdev);
1598 	struct radeon_cac_tdp_table *cac_tdp_table =
1599 		rdev->pm.dpm.dyn_state.cac_tdp_table;
1600 	u32 power_limit;
1601 
1602 	if (ac_power)
1603 		power_limit = (u32)(cac_tdp_table->maximum_power_delivery_limit * 256);
1604 	else
1605 		power_limit = (u32)(cac_tdp_table->battery_power_limit * 256);
1606 
1607 	ci_set_power_limit(rdev, power_limit);
1608 
1609 	if (pi->caps_automatic_dc_transition) {
1610 		if (ac_power)
1611 			ci_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC);
1612 		else
1613 			ci_send_msg_to_smc(rdev, PPSMC_MSG_Remove_DC_Clamp);
1614 	}
1615 
1616 	return 0;
1617 }
1618 #endif
1619 
ci_send_msg_to_smc(struct radeon_device * rdev,PPSMC_Msg msg)1620 static PPSMC_Result ci_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg)
1621 {
1622 	u32 tmp;
1623 	int i;
1624 
1625 	if (!ci_is_smc_running(rdev))
1626 		return PPSMC_Result_Failed;
1627 
1628 	WREG32(SMC_MESSAGE_0, msg);
1629 
1630 	for (i = 0; i < rdev->usec_timeout; i++) {
1631 		tmp = RREG32(SMC_RESP_0);
1632 		if (tmp != 0)
1633 			break;
1634 		udelay(1);
1635 	}
1636 	tmp = RREG32(SMC_RESP_0);
1637 
1638 	return (PPSMC_Result)tmp;
1639 }
1640 
ci_send_msg_to_smc_with_parameter(struct radeon_device * rdev,PPSMC_Msg msg,u32 parameter)1641 static PPSMC_Result ci_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1642 						      PPSMC_Msg msg, u32 parameter)
1643 {
1644 	WREG32(SMC_MSG_ARG_0, parameter);
1645 	return ci_send_msg_to_smc(rdev, msg);
1646 }
1647 
ci_send_msg_to_smc_return_parameter(struct radeon_device * rdev,PPSMC_Msg msg,u32 * parameter)1648 static PPSMC_Result ci_send_msg_to_smc_return_parameter(struct radeon_device *rdev,
1649 							PPSMC_Msg msg, u32 *parameter)
1650 {
1651 	PPSMC_Result smc_result;
1652 
1653 	smc_result = ci_send_msg_to_smc(rdev, msg);
1654 
1655 	if ((smc_result == PPSMC_Result_OK) && parameter)
1656 		*parameter = RREG32(SMC_MSG_ARG_0);
1657 
1658 	return smc_result;
1659 }
1660 
ci_dpm_force_state_sclk(struct radeon_device * rdev,u32 n)1661 static int ci_dpm_force_state_sclk(struct radeon_device *rdev, u32 n)
1662 {
1663 	struct ci_power_info *pi = ci_get_pi(rdev);
1664 
1665 	if (!pi->sclk_dpm_key_disabled) {
1666 		PPSMC_Result smc_result =
1667 			ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SCLKDPM_SetEnabledMask, 1 << n);
1668 		if (smc_result != PPSMC_Result_OK)
1669 			return -EINVAL;
1670 	}
1671 
1672 	return 0;
1673 }
1674 
ci_dpm_force_state_mclk(struct radeon_device * rdev,u32 n)1675 static int ci_dpm_force_state_mclk(struct radeon_device *rdev, u32 n)
1676 {
1677 	struct ci_power_info *pi = ci_get_pi(rdev);
1678 
1679 	if (!pi->mclk_dpm_key_disabled) {
1680 		PPSMC_Result smc_result =
1681 			ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_MCLKDPM_SetEnabledMask, 1 << n);
1682 		if (smc_result != PPSMC_Result_OK)
1683 			return -EINVAL;
1684 	}
1685 
1686 	return 0;
1687 }
1688 
ci_dpm_force_state_pcie(struct radeon_device * rdev,u32 n)1689 static int ci_dpm_force_state_pcie(struct radeon_device *rdev, u32 n)
1690 {
1691 	struct ci_power_info *pi = ci_get_pi(rdev);
1692 
1693 	if (!pi->pcie_dpm_key_disabled) {
1694 		PPSMC_Result smc_result =
1695 			ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_PCIeDPM_ForceLevel, n);
1696 		if (smc_result != PPSMC_Result_OK)
1697 			return -EINVAL;
1698 	}
1699 
1700 	return 0;
1701 }
1702 
ci_set_power_limit(struct radeon_device * rdev,u32 n)1703 static int ci_set_power_limit(struct radeon_device *rdev, u32 n)
1704 {
1705 	struct ci_power_info *pi = ci_get_pi(rdev);
1706 
1707 	if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit) {
1708 		PPSMC_Result smc_result =
1709 			ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_PkgPwrSetLimit, n);
1710 		if (smc_result != PPSMC_Result_OK)
1711 			return -EINVAL;
1712 	}
1713 
1714 	return 0;
1715 }
1716 
ci_set_overdrive_target_tdp(struct radeon_device * rdev,u32 target_tdp)1717 static int ci_set_overdrive_target_tdp(struct radeon_device *rdev,
1718 				       u32 target_tdp)
1719 {
1720 	PPSMC_Result smc_result =
1721 		ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_OverDriveSetTargetTdp, target_tdp);
1722 	if (smc_result != PPSMC_Result_OK)
1723 		return -EINVAL;
1724 	return 0;
1725 }
1726 
1727 #if 0
1728 static int ci_set_boot_state(struct radeon_device *rdev)
1729 {
1730 	return ci_enable_sclk_mclk_dpm(rdev, false);
1731 }
1732 #endif
1733 
ci_get_average_sclk_freq(struct radeon_device * rdev)1734 static u32 ci_get_average_sclk_freq(struct radeon_device *rdev)
1735 {
1736 	u32 sclk_freq;
1737 	PPSMC_Result smc_result =
1738 		ci_send_msg_to_smc_return_parameter(rdev,
1739 						    PPSMC_MSG_API_GetSclkFrequency,
1740 						    &sclk_freq);
1741 	if (smc_result != PPSMC_Result_OK)
1742 		sclk_freq = 0;
1743 
1744 	return sclk_freq;
1745 }
1746 
ci_get_average_mclk_freq(struct radeon_device * rdev)1747 static u32 ci_get_average_mclk_freq(struct radeon_device *rdev)
1748 {
1749 	u32 mclk_freq;
1750 	PPSMC_Result smc_result =
1751 		ci_send_msg_to_smc_return_parameter(rdev,
1752 						    PPSMC_MSG_API_GetMclkFrequency,
1753 						    &mclk_freq);
1754 	if (smc_result != PPSMC_Result_OK)
1755 		mclk_freq = 0;
1756 
1757 	return mclk_freq;
1758 }
1759 
ci_dpm_start_smc(struct radeon_device * rdev)1760 static void ci_dpm_start_smc(struct radeon_device *rdev)
1761 {
1762 	int i;
1763 
1764 	ci_program_jump_on_start(rdev);
1765 	ci_start_smc_clock(rdev);
1766 	ci_start_smc(rdev);
1767 	for (i = 0; i < rdev->usec_timeout; i++) {
1768 		if (RREG32_SMC(FIRMWARE_FLAGS) & INTERRUPTS_ENABLED)
1769 			break;
1770 	}
1771 }
1772 
ci_dpm_stop_smc(struct radeon_device * rdev)1773 static void ci_dpm_stop_smc(struct radeon_device *rdev)
1774 {
1775 	ci_reset_smc(rdev);
1776 	ci_stop_smc_clock(rdev);
1777 }
1778 
ci_process_firmware_header(struct radeon_device * rdev)1779 static int ci_process_firmware_header(struct radeon_device *rdev)
1780 {
1781 	struct ci_power_info *pi = ci_get_pi(rdev);
1782 	u32 tmp;
1783 	int ret;
1784 
1785 	ret = ci_read_smc_sram_dword(rdev,
1786 				     SMU7_FIRMWARE_HEADER_LOCATION +
1787 				     offsetof(SMU7_Firmware_Header, DpmTable),
1788 				     &tmp, pi->sram_end);
1789 	if (ret)
1790 		return ret;
1791 
1792 	pi->dpm_table_start = tmp;
1793 
1794 	ret = ci_read_smc_sram_dword(rdev,
1795 				     SMU7_FIRMWARE_HEADER_LOCATION +
1796 				     offsetof(SMU7_Firmware_Header, SoftRegisters),
1797 				     &tmp, pi->sram_end);
1798 	if (ret)
1799 		return ret;
1800 
1801 	pi->soft_regs_start = tmp;
1802 
1803 	ret = ci_read_smc_sram_dword(rdev,
1804 				     SMU7_FIRMWARE_HEADER_LOCATION +
1805 				     offsetof(SMU7_Firmware_Header, mcRegisterTable),
1806 				     &tmp, pi->sram_end);
1807 	if (ret)
1808 		return ret;
1809 
1810 	pi->mc_reg_table_start = tmp;
1811 
1812 	ret = ci_read_smc_sram_dword(rdev,
1813 				     SMU7_FIRMWARE_HEADER_LOCATION +
1814 				     offsetof(SMU7_Firmware_Header, FanTable),
1815 				     &tmp, pi->sram_end);
1816 	if (ret)
1817 		return ret;
1818 
1819 	pi->fan_table_start = tmp;
1820 
1821 	ret = ci_read_smc_sram_dword(rdev,
1822 				     SMU7_FIRMWARE_HEADER_LOCATION +
1823 				     offsetof(SMU7_Firmware_Header, mcArbDramTimingTable),
1824 				     &tmp, pi->sram_end);
1825 	if (ret)
1826 		return ret;
1827 
1828 	pi->arb_table_start = tmp;
1829 
1830 	return 0;
1831 }
1832 
ci_read_clock_registers(struct radeon_device * rdev)1833 static void ci_read_clock_registers(struct radeon_device *rdev)
1834 {
1835 	struct ci_power_info *pi = ci_get_pi(rdev);
1836 
1837 	pi->clock_registers.cg_spll_func_cntl =
1838 		RREG32_SMC(CG_SPLL_FUNC_CNTL);
1839 	pi->clock_registers.cg_spll_func_cntl_2 =
1840 		RREG32_SMC(CG_SPLL_FUNC_CNTL_2);
1841 	pi->clock_registers.cg_spll_func_cntl_3 =
1842 		RREG32_SMC(CG_SPLL_FUNC_CNTL_3);
1843 	pi->clock_registers.cg_spll_func_cntl_4 =
1844 		RREG32_SMC(CG_SPLL_FUNC_CNTL_4);
1845 	pi->clock_registers.cg_spll_spread_spectrum =
1846 		RREG32_SMC(CG_SPLL_SPREAD_SPECTRUM);
1847 	pi->clock_registers.cg_spll_spread_spectrum_2 =
1848 		RREG32_SMC(CG_SPLL_SPREAD_SPECTRUM_2);
1849 	pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1850 	pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1851 	pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1852 	pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1853 	pi->clock_registers.mpll_func_cntl = RREG32(MPLL_FUNC_CNTL);
1854 	pi->clock_registers.mpll_func_cntl_1 = RREG32(MPLL_FUNC_CNTL_1);
1855 	pi->clock_registers.mpll_func_cntl_2 = RREG32(MPLL_FUNC_CNTL_2);
1856 	pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1857 	pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1858 }
1859 
ci_init_sclk_t(struct radeon_device * rdev)1860 static void ci_init_sclk_t(struct radeon_device *rdev)
1861 {
1862 	struct ci_power_info *pi = ci_get_pi(rdev);
1863 
1864 	pi->low_sclk_interrupt_t = 0;
1865 }
1866 
ci_enable_thermal_protection(struct radeon_device * rdev,bool enable)1867 static void ci_enable_thermal_protection(struct radeon_device *rdev,
1868 					 bool enable)
1869 {
1870 	u32 tmp = RREG32_SMC(GENERAL_PWRMGT);
1871 
1872 	if (enable)
1873 		tmp &= ~THERMAL_PROTECTION_DIS;
1874 	else
1875 		tmp |= THERMAL_PROTECTION_DIS;
1876 	WREG32_SMC(GENERAL_PWRMGT, tmp);
1877 }
1878 
ci_enable_acpi_power_management(struct radeon_device * rdev)1879 static void ci_enable_acpi_power_management(struct radeon_device *rdev)
1880 {
1881 	u32 tmp = RREG32_SMC(GENERAL_PWRMGT);
1882 
1883 	tmp |= STATIC_PM_EN;
1884 
1885 	WREG32_SMC(GENERAL_PWRMGT, tmp);
1886 }
1887 
1888 #if 0
1889 static int ci_enter_ulp_state(struct radeon_device *rdev)
1890 {
1891 
1892 	WREG32(SMC_MESSAGE_0, PPSMC_MSG_SwitchToMinimumPower);
1893 
1894 	udelay(25000);
1895 
1896 	return 0;
1897 }
1898 
1899 static int ci_exit_ulp_state(struct radeon_device *rdev)
1900 {
1901 	int i;
1902 
1903 	WREG32(SMC_MESSAGE_0, PPSMC_MSG_ResumeFromMinimumPower);
1904 
1905 	udelay(7000);
1906 
1907 	for (i = 0; i < rdev->usec_timeout; i++) {
1908 		if (RREG32(SMC_RESP_0) == 1)
1909 			break;
1910 		udelay(1000);
1911 	}
1912 
1913 	return 0;
1914 }
1915 #endif
1916 
ci_notify_smc_display_change(struct radeon_device * rdev,bool has_display)1917 static int ci_notify_smc_display_change(struct radeon_device *rdev,
1918 					bool has_display)
1919 {
1920 	PPSMC_Msg msg = has_display ? PPSMC_MSG_HasDisplay : PPSMC_MSG_NoDisplay;
1921 
1922 	return (ci_send_msg_to_smc(rdev, msg) == PPSMC_Result_OK) ?  0 : -EINVAL;
1923 }
1924 
ci_enable_ds_master_switch(struct radeon_device * rdev,bool enable)1925 static int ci_enable_ds_master_switch(struct radeon_device *rdev,
1926 				      bool enable)
1927 {
1928 	struct ci_power_info *pi = ci_get_pi(rdev);
1929 
1930 	if (enable) {
1931 		if (pi->caps_sclk_ds) {
1932 			if (ci_send_msg_to_smc(rdev, PPSMC_MSG_MASTER_DeepSleep_ON) != PPSMC_Result_OK)
1933 				return -EINVAL;
1934 		} else {
1935 			if (ci_send_msg_to_smc(rdev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK)
1936 				return -EINVAL;
1937 		}
1938 	} else {
1939 		if (pi->caps_sclk_ds) {
1940 			if (ci_send_msg_to_smc(rdev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK)
1941 				return -EINVAL;
1942 		}
1943 	}
1944 
1945 	return 0;
1946 }
1947 
ci_program_display_gap(struct radeon_device * rdev)1948 static void ci_program_display_gap(struct radeon_device *rdev)
1949 {
1950 	u32 tmp = RREG32_SMC(CG_DISPLAY_GAP_CNTL);
1951 	u32 pre_vbi_time_in_us;
1952 	u32 frame_time_in_us;
1953 	u32 ref_clock = rdev->clock.spll.reference_freq;
1954 	u32 refresh_rate = r600_dpm_get_vrefresh(rdev);
1955 	u32 vblank_time = r600_dpm_get_vblank_time(rdev);
1956 
1957 	tmp &= ~DISP_GAP_MASK;
1958 	if (rdev->pm.dpm.new_active_crtc_count > 0)
1959 		tmp |= DISP_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM);
1960 	else
1961 		tmp |= DISP_GAP(R600_PM_DISPLAY_GAP_IGNORE);
1962 	WREG32_SMC(CG_DISPLAY_GAP_CNTL, tmp);
1963 
1964 	if (refresh_rate == 0)
1965 		refresh_rate = 60;
1966 	if (vblank_time == 0xffffffff)
1967 		vblank_time = 500;
1968 	frame_time_in_us = 1000000 / refresh_rate;
1969 	pre_vbi_time_in_us =
1970 		frame_time_in_us - 200 - vblank_time;
1971 	tmp = pre_vbi_time_in_us * (ref_clock / 100);
1972 
1973 	WREG32_SMC(CG_DISPLAY_GAP_CNTL2, tmp);
1974 	ci_write_smc_soft_register(rdev, offsetof(SMU7_SoftRegisters, PreVBlankGap), 0x64);
1975 	ci_write_smc_soft_register(rdev, offsetof(SMU7_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us));
1976 
1977 
1978 	ci_notify_smc_display_change(rdev, (rdev->pm.dpm.new_active_crtc_count == 1));
1979 
1980 }
1981 
ci_enable_spread_spectrum(struct radeon_device * rdev,bool enable)1982 static void ci_enable_spread_spectrum(struct radeon_device *rdev, bool enable)
1983 {
1984 	struct ci_power_info *pi = ci_get_pi(rdev);
1985 	u32 tmp;
1986 
1987 	if (enable) {
1988 		if (pi->caps_sclk_ss_support) {
1989 			tmp = RREG32_SMC(GENERAL_PWRMGT);
1990 			tmp |= DYN_SPREAD_SPECTRUM_EN;
1991 			WREG32_SMC(GENERAL_PWRMGT, tmp);
1992 		}
1993 	} else {
1994 		tmp = RREG32_SMC(CG_SPLL_SPREAD_SPECTRUM);
1995 		tmp &= ~SSEN;
1996 		WREG32_SMC(CG_SPLL_SPREAD_SPECTRUM, tmp);
1997 
1998 		tmp = RREG32_SMC(GENERAL_PWRMGT);
1999 		tmp &= ~DYN_SPREAD_SPECTRUM_EN;
2000 		WREG32_SMC(GENERAL_PWRMGT, tmp);
2001 	}
2002 }
2003 
ci_program_sstp(struct radeon_device * rdev)2004 static void ci_program_sstp(struct radeon_device *rdev)
2005 {
2006 	WREG32_SMC(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT)));
2007 }
2008 
ci_enable_display_gap(struct radeon_device * rdev)2009 static void ci_enable_display_gap(struct radeon_device *rdev)
2010 {
2011 	u32 tmp = RREG32_SMC(CG_DISPLAY_GAP_CNTL);
2012 
2013 	tmp &= ~(DISP_GAP_MASK | DISP_GAP_MCHG_MASK);
2014 	tmp |= (DISP_GAP(R600_PM_DISPLAY_GAP_IGNORE) |
2015 		DISP_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK));
2016 
2017 	WREG32_SMC(CG_DISPLAY_GAP_CNTL, tmp);
2018 }
2019 
ci_program_vc(struct radeon_device * rdev)2020 static void ci_program_vc(struct radeon_device *rdev)
2021 {
2022 	u32 tmp;
2023 
2024 	tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
2025 	tmp &= ~(RESET_SCLK_CNT | RESET_BUSY_CNT);
2026 	WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
2027 
2028 	WREG32_SMC(CG_FTV_0, CISLANDS_VRC_DFLT0);
2029 	WREG32_SMC(CG_FTV_1, CISLANDS_VRC_DFLT1);
2030 	WREG32_SMC(CG_FTV_2, CISLANDS_VRC_DFLT2);
2031 	WREG32_SMC(CG_FTV_3, CISLANDS_VRC_DFLT3);
2032 	WREG32_SMC(CG_FTV_4, CISLANDS_VRC_DFLT4);
2033 	WREG32_SMC(CG_FTV_5, CISLANDS_VRC_DFLT5);
2034 	WREG32_SMC(CG_FTV_6, CISLANDS_VRC_DFLT6);
2035 	WREG32_SMC(CG_FTV_7, CISLANDS_VRC_DFLT7);
2036 }
2037 
ci_clear_vc(struct radeon_device * rdev)2038 static void ci_clear_vc(struct radeon_device *rdev)
2039 {
2040 	u32 tmp;
2041 
2042 	tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
2043 	tmp |= (RESET_SCLK_CNT | RESET_BUSY_CNT);
2044 	WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
2045 
2046 	WREG32_SMC(CG_FTV_0, 0);
2047 	WREG32_SMC(CG_FTV_1, 0);
2048 	WREG32_SMC(CG_FTV_2, 0);
2049 	WREG32_SMC(CG_FTV_3, 0);
2050 	WREG32_SMC(CG_FTV_4, 0);
2051 	WREG32_SMC(CG_FTV_5, 0);
2052 	WREG32_SMC(CG_FTV_6, 0);
2053 	WREG32_SMC(CG_FTV_7, 0);
2054 }
2055 
ci_upload_firmware(struct radeon_device * rdev)2056 static int ci_upload_firmware(struct radeon_device *rdev)
2057 {
2058 	struct ci_power_info *pi = ci_get_pi(rdev);
2059 	int i;
2060 
2061 	for (i = 0; i < rdev->usec_timeout; i++) {
2062 		if (RREG32_SMC(RCU_UC_EVENTS) & BOOT_SEQ_DONE)
2063 			break;
2064 	}
2065 	WREG32_SMC(SMC_SYSCON_MISC_CNTL, 1);
2066 
2067 	ci_stop_smc_clock(rdev);
2068 	ci_reset_smc(rdev);
2069 
2070 	return ci_load_smc_ucode(rdev, pi->sram_end);
2071 
2072 }
2073 
ci_get_svi2_voltage_table(struct radeon_device * rdev,struct radeon_clock_voltage_dependency_table * voltage_dependency_table,struct atom_voltage_table * voltage_table)2074 static int ci_get_svi2_voltage_table(struct radeon_device *rdev,
2075 				     struct radeon_clock_voltage_dependency_table *voltage_dependency_table,
2076 				     struct atom_voltage_table *voltage_table)
2077 {
2078 	u32 i;
2079 
2080 	if (voltage_dependency_table == NULL)
2081 		return -EINVAL;
2082 
2083 	voltage_table->mask_low = 0;
2084 	voltage_table->phase_delay = 0;
2085 
2086 	voltage_table->count = voltage_dependency_table->count;
2087 	for (i = 0; i < voltage_table->count; i++) {
2088 		voltage_table->entries[i].value = voltage_dependency_table->entries[i].v;
2089 		voltage_table->entries[i].smio_low = 0;
2090 	}
2091 
2092 	return 0;
2093 }
2094 
ci_construct_voltage_tables(struct radeon_device * rdev)2095 static int ci_construct_voltage_tables(struct radeon_device *rdev)
2096 {
2097 	struct ci_power_info *pi = ci_get_pi(rdev);
2098 	int ret;
2099 
2100 	if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) {
2101 		ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDC,
2102 						    VOLTAGE_OBJ_GPIO_LUT,
2103 						    &pi->vddc_voltage_table);
2104 		if (ret)
2105 			return ret;
2106 	} else if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
2107 		ret = ci_get_svi2_voltage_table(rdev,
2108 						&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
2109 						&pi->vddc_voltage_table);
2110 		if (ret)
2111 			return ret;
2112 	}
2113 
2114 	if (pi->vddc_voltage_table.count > SMU7_MAX_LEVELS_VDDC)
2115 		si_trim_voltage_table_to_fit_state_table(rdev, SMU7_MAX_LEVELS_VDDC,
2116 							 &pi->vddc_voltage_table);
2117 
2118 	if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) {
2119 		ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDCI,
2120 						    VOLTAGE_OBJ_GPIO_LUT,
2121 						    &pi->vddci_voltage_table);
2122 		if (ret)
2123 			return ret;
2124 	} else if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
2125 		ret = ci_get_svi2_voltage_table(rdev,
2126 						&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
2127 						&pi->vddci_voltage_table);
2128 		if (ret)
2129 			return ret;
2130 	}
2131 
2132 	if (pi->vddci_voltage_table.count > SMU7_MAX_LEVELS_VDDCI)
2133 		si_trim_voltage_table_to_fit_state_table(rdev, SMU7_MAX_LEVELS_VDDCI,
2134 							 &pi->vddci_voltage_table);
2135 
2136 	if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) {
2137 		ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_MVDDC,
2138 						    VOLTAGE_OBJ_GPIO_LUT,
2139 						    &pi->mvdd_voltage_table);
2140 		if (ret)
2141 			return ret;
2142 	} else if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
2143 		ret = ci_get_svi2_voltage_table(rdev,
2144 						&rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk,
2145 						&pi->mvdd_voltage_table);
2146 		if (ret)
2147 			return ret;
2148 	}
2149 
2150 	if (pi->mvdd_voltage_table.count > SMU7_MAX_LEVELS_MVDD)
2151 		si_trim_voltage_table_to_fit_state_table(rdev, SMU7_MAX_LEVELS_MVDD,
2152 							 &pi->mvdd_voltage_table);
2153 
2154 	return 0;
2155 }
2156 
ci_populate_smc_voltage_table(struct radeon_device * rdev,struct atom_voltage_table_entry * voltage_table,SMU7_Discrete_VoltageLevel * smc_voltage_table)2157 static void ci_populate_smc_voltage_table(struct radeon_device *rdev,
2158 					  struct atom_voltage_table_entry *voltage_table,
2159 					  SMU7_Discrete_VoltageLevel *smc_voltage_table)
2160 {
2161 	int ret;
2162 
2163 	ret = ci_get_std_voltage_value_sidd(rdev, voltage_table,
2164 					    &smc_voltage_table->StdVoltageHiSidd,
2165 					    &smc_voltage_table->StdVoltageLoSidd);
2166 
2167 	if (ret) {
2168 		smc_voltage_table->StdVoltageHiSidd = voltage_table->value * VOLTAGE_SCALE;
2169 		smc_voltage_table->StdVoltageLoSidd = voltage_table->value * VOLTAGE_SCALE;
2170 	}
2171 
2172 	smc_voltage_table->Voltage = cpu_to_be16(voltage_table->value * VOLTAGE_SCALE);
2173 	smc_voltage_table->StdVoltageHiSidd =
2174 		cpu_to_be16(smc_voltage_table->StdVoltageHiSidd);
2175 	smc_voltage_table->StdVoltageLoSidd =
2176 		cpu_to_be16(smc_voltage_table->StdVoltageLoSidd);
2177 }
2178 
ci_populate_smc_vddc_table(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2179 static int ci_populate_smc_vddc_table(struct radeon_device *rdev,
2180 				      SMU7_Discrete_DpmTable *table)
2181 {
2182 	struct ci_power_info *pi = ci_get_pi(rdev);
2183 	unsigned int count;
2184 
2185 	table->VddcLevelCount = pi->vddc_voltage_table.count;
2186 	for (count = 0; count < table->VddcLevelCount; count++) {
2187 		ci_populate_smc_voltage_table(rdev,
2188 					      &pi->vddc_voltage_table.entries[count],
2189 					      &table->VddcLevel[count]);
2190 
2191 		if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO)
2192 			table->VddcLevel[count].Smio |=
2193 				pi->vddc_voltage_table.entries[count].smio_low;
2194 		else
2195 			table->VddcLevel[count].Smio = 0;
2196 	}
2197 	table->VddcLevelCount = cpu_to_be32(table->VddcLevelCount);
2198 
2199 	return 0;
2200 }
2201 
ci_populate_smc_vddci_table(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2202 static int ci_populate_smc_vddci_table(struct radeon_device *rdev,
2203 				       SMU7_Discrete_DpmTable *table)
2204 {
2205 	unsigned int count;
2206 	struct ci_power_info *pi = ci_get_pi(rdev);
2207 
2208 	table->VddciLevelCount = pi->vddci_voltage_table.count;
2209 	for (count = 0; count < table->VddciLevelCount; count++) {
2210 		ci_populate_smc_voltage_table(rdev,
2211 					      &pi->vddci_voltage_table.entries[count],
2212 					      &table->VddciLevel[count]);
2213 
2214 		if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO)
2215 			table->VddciLevel[count].Smio |=
2216 				pi->vddci_voltage_table.entries[count].smio_low;
2217 		else
2218 			table->VddciLevel[count].Smio = 0;
2219 	}
2220 	table->VddciLevelCount = cpu_to_be32(table->VddciLevelCount);
2221 
2222 	return 0;
2223 }
2224 
ci_populate_smc_mvdd_table(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2225 static int ci_populate_smc_mvdd_table(struct radeon_device *rdev,
2226 				      SMU7_Discrete_DpmTable *table)
2227 {
2228 	struct ci_power_info *pi = ci_get_pi(rdev);
2229 	unsigned int count;
2230 
2231 	table->MvddLevelCount = pi->mvdd_voltage_table.count;
2232 	for (count = 0; count < table->MvddLevelCount; count++) {
2233 		ci_populate_smc_voltage_table(rdev,
2234 					      &pi->mvdd_voltage_table.entries[count],
2235 					      &table->MvddLevel[count]);
2236 
2237 		if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO)
2238 			table->MvddLevel[count].Smio |=
2239 				pi->mvdd_voltage_table.entries[count].smio_low;
2240 		else
2241 			table->MvddLevel[count].Smio = 0;
2242 	}
2243 	table->MvddLevelCount = cpu_to_be32(table->MvddLevelCount);
2244 
2245 	return 0;
2246 }
2247 
ci_populate_smc_voltage_tables(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2248 static int ci_populate_smc_voltage_tables(struct radeon_device *rdev,
2249 					  SMU7_Discrete_DpmTable *table)
2250 {
2251 	int ret;
2252 
2253 	ret = ci_populate_smc_vddc_table(rdev, table);
2254 	if (ret)
2255 		return ret;
2256 
2257 	ret = ci_populate_smc_vddci_table(rdev, table);
2258 	if (ret)
2259 		return ret;
2260 
2261 	ret = ci_populate_smc_mvdd_table(rdev, table);
2262 	if (ret)
2263 		return ret;
2264 
2265 	return 0;
2266 }
2267 
ci_populate_mvdd_value(struct radeon_device * rdev,u32 mclk,SMU7_Discrete_VoltageLevel * voltage)2268 static int ci_populate_mvdd_value(struct radeon_device *rdev, u32 mclk,
2269 				  SMU7_Discrete_VoltageLevel *voltage)
2270 {
2271 	struct ci_power_info *pi = ci_get_pi(rdev);
2272 	u32 i = 0;
2273 
2274 	if (pi->mvdd_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
2275 		for (i = 0; i < rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count; i++) {
2276 			if (mclk <= rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries[i].clk) {
2277 				voltage->Voltage = pi->mvdd_voltage_table.entries[i].value;
2278 				break;
2279 			}
2280 		}
2281 
2282 		if (i >= rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count)
2283 			return -EINVAL;
2284 	}
2285 
2286 	return -EINVAL;
2287 }
2288 
ci_get_std_voltage_value_sidd(struct radeon_device * rdev,struct atom_voltage_table_entry * voltage_table,u16 * std_voltage_hi_sidd,u16 * std_voltage_lo_sidd)2289 static int ci_get_std_voltage_value_sidd(struct radeon_device *rdev,
2290 					 struct atom_voltage_table_entry *voltage_table,
2291 					 u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd)
2292 {
2293 	u16 v_index, idx;
2294 	bool voltage_found = false;
2295 	*std_voltage_hi_sidd = voltage_table->value * VOLTAGE_SCALE;
2296 	*std_voltage_lo_sidd = voltage_table->value * VOLTAGE_SCALE;
2297 
2298 	if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries == NULL)
2299 		return -EINVAL;
2300 
2301 	if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries) {
2302 		for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
2303 			if (voltage_table->value ==
2304 			    rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
2305 				voltage_found = true;
2306 				if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count)
2307 					idx = v_index;
2308 				else
2309 					idx = rdev->pm.dpm.dyn_state.cac_leakage_table.count - 1;
2310 				*std_voltage_lo_sidd =
2311 					rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE;
2312 				*std_voltage_hi_sidd =
2313 					rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE;
2314 				break;
2315 			}
2316 		}
2317 
2318 		if (!voltage_found) {
2319 			for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
2320 				if (voltage_table->value <=
2321 				    rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
2322 					voltage_found = true;
2323 					if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count)
2324 						idx = v_index;
2325 					else
2326 						idx = rdev->pm.dpm.dyn_state.cac_leakage_table.count - 1;
2327 					*std_voltage_lo_sidd =
2328 						rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE;
2329 					*std_voltage_hi_sidd =
2330 						rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE;
2331 					break;
2332 				}
2333 			}
2334 		}
2335 	}
2336 
2337 	return 0;
2338 }
2339 
ci_populate_phase_value_based_on_sclk(struct radeon_device * rdev,const struct radeon_phase_shedding_limits_table * limits,u32 sclk,u32 * phase_shedding)2340 static void ci_populate_phase_value_based_on_sclk(struct radeon_device *rdev,
2341 						  const struct radeon_phase_shedding_limits_table *limits,
2342 						  u32 sclk,
2343 						  u32 *phase_shedding)
2344 {
2345 	unsigned int i;
2346 
2347 	*phase_shedding = 1;
2348 
2349 	for (i = 0; i < limits->count; i++) {
2350 		if (sclk < limits->entries[i].sclk) {
2351 			*phase_shedding = i;
2352 			break;
2353 		}
2354 	}
2355 }
2356 
ci_populate_phase_value_based_on_mclk(struct radeon_device * rdev,const struct radeon_phase_shedding_limits_table * limits,u32 mclk,u32 * phase_shedding)2357 static void ci_populate_phase_value_based_on_mclk(struct radeon_device *rdev,
2358 						  const struct radeon_phase_shedding_limits_table *limits,
2359 						  u32 mclk,
2360 						  u32 *phase_shedding)
2361 {
2362 	unsigned int i;
2363 
2364 	*phase_shedding = 1;
2365 
2366 	for (i = 0; i < limits->count; i++) {
2367 		if (mclk < limits->entries[i].mclk) {
2368 			*phase_shedding = i;
2369 			break;
2370 		}
2371 	}
2372 }
2373 
ci_init_arb_table_index(struct radeon_device * rdev)2374 static int ci_init_arb_table_index(struct radeon_device *rdev)
2375 {
2376 	struct ci_power_info *pi = ci_get_pi(rdev);
2377 	u32 tmp;
2378 	int ret;
2379 
2380 	ret = ci_read_smc_sram_dword(rdev, pi->arb_table_start,
2381 				     &tmp, pi->sram_end);
2382 	if (ret)
2383 		return ret;
2384 
2385 	tmp &= 0x00FFFFFF;
2386 	tmp |= MC_CG_ARB_FREQ_F1 << 24;
2387 
2388 	return ci_write_smc_sram_dword(rdev, pi->arb_table_start,
2389 				       tmp, pi->sram_end);
2390 }
2391 
ci_get_dependency_volt_by_clk(struct radeon_device * rdev,struct radeon_clock_voltage_dependency_table * allowed_clock_voltage_table,u32 clock,u32 * voltage)2392 static int ci_get_dependency_volt_by_clk(struct radeon_device *rdev,
2393 					 struct radeon_clock_voltage_dependency_table *allowed_clock_voltage_table,
2394 					 u32 clock, u32 *voltage)
2395 {
2396 	u32 i = 0;
2397 
2398 	if (allowed_clock_voltage_table->count == 0)
2399 		return -EINVAL;
2400 
2401 	for (i = 0; i < allowed_clock_voltage_table->count; i++) {
2402 		if (allowed_clock_voltage_table->entries[i].clk >= clock) {
2403 			*voltage = allowed_clock_voltage_table->entries[i].v;
2404 			return 0;
2405 		}
2406 	}
2407 
2408 	*voltage = allowed_clock_voltage_table->entries[i-1].v;
2409 
2410 	return 0;
2411 }
2412 
ci_get_sleep_divider_id_from_clock(struct radeon_device * rdev,u32 sclk,u32 min_sclk_in_sr)2413 static u8 ci_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
2414 					     u32 sclk, u32 min_sclk_in_sr)
2415 {
2416 	u32 i;
2417 	u32 tmp;
2418 	u32 min = (min_sclk_in_sr > CISLAND_MINIMUM_ENGINE_CLOCK) ?
2419 		min_sclk_in_sr : CISLAND_MINIMUM_ENGINE_CLOCK;
2420 
2421 	if (sclk < min)
2422 		return 0;
2423 
2424 	for (i = CISLAND_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
2425 		tmp = sclk / (1 << i);
2426 		if (tmp >= min || i == 0)
2427 			break;
2428 	}
2429 
2430 	return (u8)i;
2431 }
2432 
ci_initial_switch_from_arb_f0_to_f1(struct radeon_device * rdev)2433 static int ci_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
2434 {
2435 	return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
2436 }
2437 
ci_reset_to_default(struct radeon_device * rdev)2438 static int ci_reset_to_default(struct radeon_device *rdev)
2439 {
2440 	return (ci_send_msg_to_smc(rdev, PPSMC_MSG_ResetToDefaults) == PPSMC_Result_OK) ?
2441 		0 : -EINVAL;
2442 }
2443 
ci_force_switch_to_arb_f0(struct radeon_device * rdev)2444 static int ci_force_switch_to_arb_f0(struct radeon_device *rdev)
2445 {
2446 	u32 tmp;
2447 
2448 	tmp = (RREG32_SMC(SMC_SCRATCH9) & 0x0000ff00) >> 8;
2449 
2450 	if (tmp == MC_CG_ARB_FREQ_F0)
2451 		return 0;
2452 
2453 	return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
2454 }
2455 
ci_register_patching_mc_arb(struct radeon_device * rdev,const u32 engine_clock,const u32 memory_clock,u32 * dram_timimg2)2456 static void ci_register_patching_mc_arb(struct radeon_device *rdev,
2457 					const u32 engine_clock,
2458 					const u32 memory_clock,
2459 					u32 *dram_timimg2)
2460 {
2461 	bool patch;
2462 	u32 tmp, tmp2;
2463 
2464 	tmp = RREG32(MC_SEQ_MISC0);
2465 	patch = ((tmp & 0x0000f00) == 0x300) ? true : false;
2466 
2467 	if (patch &&
2468 	    ((rdev->pdev->device == 0x67B0) ||
2469 	     (rdev->pdev->device == 0x67B1))) {
2470 		if ((memory_clock > 100000) && (memory_clock <= 125000)) {
2471 			tmp2 = (((0x31 * engine_clock) / 125000) - 1) & 0xff;
2472 			*dram_timimg2 &= ~0x00ff0000;
2473 			*dram_timimg2 |= tmp2 << 16;
2474 		} else if ((memory_clock > 125000) && (memory_clock <= 137500)) {
2475 			tmp2 = (((0x36 * engine_clock) / 137500) - 1) & 0xff;
2476 			*dram_timimg2 &= ~0x00ff0000;
2477 			*dram_timimg2 |= tmp2 << 16;
2478 		}
2479 	}
2480 }
2481 
2482 
ci_populate_memory_timing_parameters(struct radeon_device * rdev,u32 sclk,u32 mclk,SMU7_Discrete_MCArbDramTimingTableEntry * arb_regs)2483 static int ci_populate_memory_timing_parameters(struct radeon_device *rdev,
2484 						u32 sclk,
2485 						u32 mclk,
2486 						SMU7_Discrete_MCArbDramTimingTableEntry *arb_regs)
2487 {
2488 	u32 dram_timing;
2489 	u32 dram_timing2;
2490 	u32 burst_time;
2491 
2492 	radeon_atom_set_engine_dram_timings(rdev, sclk, mclk);
2493 
2494 	dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
2495 	dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
2496 	burst_time = RREG32(MC_ARB_BURST_TIME) & STATE0_MASK;
2497 
2498 	ci_register_patching_mc_arb(rdev, sclk, mclk, &dram_timing2);
2499 
2500 	arb_regs->McArbDramTiming  = cpu_to_be32(dram_timing);
2501 	arb_regs->McArbDramTiming2 = cpu_to_be32(dram_timing2);
2502 	arb_regs->McArbBurstTime = (u8)burst_time;
2503 
2504 	return 0;
2505 }
2506 
ci_do_program_memory_timing_parameters(struct radeon_device * rdev)2507 static int ci_do_program_memory_timing_parameters(struct radeon_device *rdev)
2508 {
2509 	struct ci_power_info *pi = ci_get_pi(rdev);
2510 	SMU7_Discrete_MCArbDramTimingTable arb_regs;
2511 	u32 i, j;
2512 	int ret =  0;
2513 
2514 	memset(&arb_regs, 0, sizeof(SMU7_Discrete_MCArbDramTimingTable));
2515 
2516 	for (i = 0; i < pi->dpm_table.sclk_table.count; i++) {
2517 		for (j = 0; j < pi->dpm_table.mclk_table.count; j++) {
2518 			ret = ci_populate_memory_timing_parameters(rdev,
2519 								   pi->dpm_table.sclk_table.dpm_levels[i].value,
2520 								   pi->dpm_table.mclk_table.dpm_levels[j].value,
2521 								   &arb_regs.entries[i][j]);
2522 			if (ret)
2523 				break;
2524 		}
2525 	}
2526 
2527 	if (ret == 0)
2528 		ret = ci_copy_bytes_to_smc(rdev,
2529 					   pi->arb_table_start,
2530 					   (u8 *)&arb_regs,
2531 					   sizeof(SMU7_Discrete_MCArbDramTimingTable),
2532 					   pi->sram_end);
2533 
2534 	return ret;
2535 }
2536 
ci_program_memory_timing_parameters(struct radeon_device * rdev)2537 static int ci_program_memory_timing_parameters(struct radeon_device *rdev)
2538 {
2539 	struct ci_power_info *pi = ci_get_pi(rdev);
2540 
2541 	if (pi->need_update_smu7_dpm_table == 0)
2542 		return 0;
2543 
2544 	return ci_do_program_memory_timing_parameters(rdev);
2545 }
2546 
ci_populate_smc_initial_state(struct radeon_device * rdev,struct radeon_ps * radeon_boot_state)2547 static void ci_populate_smc_initial_state(struct radeon_device *rdev,
2548 					  struct radeon_ps *radeon_boot_state)
2549 {
2550 	struct ci_ps *boot_state = ci_get_ps(radeon_boot_state);
2551 	struct ci_power_info *pi = ci_get_pi(rdev);
2552 	u32 level = 0;
2553 
2554 	for (level = 0; level < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; level++) {
2555 		if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[level].clk >=
2556 		    boot_state->performance_levels[0].sclk) {
2557 			pi->smc_state_table.GraphicsBootLevel = level;
2558 			break;
2559 		}
2560 	}
2561 
2562 	for (level = 0; level < rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.count; level++) {
2563 		if (rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries[level].clk >=
2564 		    boot_state->performance_levels[0].mclk) {
2565 			pi->smc_state_table.MemoryBootLevel = level;
2566 			break;
2567 		}
2568 	}
2569 }
2570 
ci_get_dpm_level_enable_mask_value(struct ci_single_dpm_table * dpm_table)2571 static u32 ci_get_dpm_level_enable_mask_value(struct ci_single_dpm_table *dpm_table)
2572 {
2573 	u32 i;
2574 	u32 mask_value = 0;
2575 
2576 	for (i = dpm_table->count; i > 0; i--) {
2577 		mask_value = mask_value << 1;
2578 		if (dpm_table->dpm_levels[i-1].enabled)
2579 			mask_value |= 0x1;
2580 		else
2581 			mask_value &= 0xFFFFFFFE;
2582 	}
2583 
2584 	return mask_value;
2585 }
2586 
ci_populate_smc_link_level(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2587 static void ci_populate_smc_link_level(struct radeon_device *rdev,
2588 				       SMU7_Discrete_DpmTable *table)
2589 {
2590 	struct ci_power_info *pi = ci_get_pi(rdev);
2591 	struct ci_dpm_table *dpm_table = &pi->dpm_table;
2592 	u32 i;
2593 
2594 	for (i = 0; i < dpm_table->pcie_speed_table.count; i++) {
2595 		table->LinkLevel[i].PcieGenSpeed =
2596 			(u8)dpm_table->pcie_speed_table.dpm_levels[i].value;
2597 		table->LinkLevel[i].PcieLaneCount =
2598 			r600_encode_pci_lane_width(dpm_table->pcie_speed_table.dpm_levels[i].param1);
2599 		table->LinkLevel[i].EnabledForActivity = 1;
2600 		table->LinkLevel[i].DownT = cpu_to_be32(5);
2601 		table->LinkLevel[i].UpT = cpu_to_be32(30);
2602 	}
2603 
2604 	pi->smc_state_table.LinkLevelCount = (u8)dpm_table->pcie_speed_table.count;
2605 	pi->dpm_level_enable_mask.pcie_dpm_enable_mask =
2606 		ci_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
2607 }
2608 
ci_populate_smc_uvd_level(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2609 static int ci_populate_smc_uvd_level(struct radeon_device *rdev,
2610 				     SMU7_Discrete_DpmTable *table)
2611 {
2612 	u32 count;
2613 	struct atom_clock_dividers dividers;
2614 	int ret = -EINVAL;
2615 
2616 	table->UvdLevelCount =
2617 		rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count;
2618 
2619 	for (count = 0; count < table->UvdLevelCount; count++) {
2620 		table->UvdLevel[count].VclkFrequency =
2621 			rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].vclk;
2622 		table->UvdLevel[count].DclkFrequency =
2623 			rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].dclk;
2624 		table->UvdLevel[count].MinVddc =
2625 			rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE;
2626 		table->UvdLevel[count].MinVddcPhases = 1;
2627 
2628 		ret = radeon_atom_get_clock_dividers(rdev,
2629 						     COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2630 						     table->UvdLevel[count].VclkFrequency, false, &dividers);
2631 		if (ret)
2632 			return ret;
2633 
2634 		table->UvdLevel[count].VclkDivider = (u8)dividers.post_divider;
2635 
2636 		ret = radeon_atom_get_clock_dividers(rdev,
2637 						     COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2638 						     table->UvdLevel[count].DclkFrequency, false, &dividers);
2639 		if (ret)
2640 			return ret;
2641 
2642 		table->UvdLevel[count].DclkDivider = (u8)dividers.post_divider;
2643 
2644 		table->UvdLevel[count].VclkFrequency = cpu_to_be32(table->UvdLevel[count].VclkFrequency);
2645 		table->UvdLevel[count].DclkFrequency = cpu_to_be32(table->UvdLevel[count].DclkFrequency);
2646 		table->UvdLevel[count].MinVddc = cpu_to_be16(table->UvdLevel[count].MinVddc);
2647 	}
2648 
2649 	return ret;
2650 }
2651 
ci_populate_smc_vce_level(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2652 static int ci_populate_smc_vce_level(struct radeon_device *rdev,
2653 				     SMU7_Discrete_DpmTable *table)
2654 {
2655 	u32 count;
2656 	struct atom_clock_dividers dividers;
2657 	int ret = -EINVAL;
2658 
2659 	table->VceLevelCount =
2660 		rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count;
2661 
2662 	for (count = 0; count < table->VceLevelCount; count++) {
2663 		table->VceLevel[count].Frequency =
2664 			rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].evclk;
2665 		table->VceLevel[count].MinVoltage =
2666 			(u16)rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE;
2667 		table->VceLevel[count].MinPhases = 1;
2668 
2669 		ret = radeon_atom_get_clock_dividers(rdev,
2670 						     COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2671 						     table->VceLevel[count].Frequency, false, &dividers);
2672 		if (ret)
2673 			return ret;
2674 
2675 		table->VceLevel[count].Divider = (u8)dividers.post_divider;
2676 
2677 		table->VceLevel[count].Frequency = cpu_to_be32(table->VceLevel[count].Frequency);
2678 		table->VceLevel[count].MinVoltage = cpu_to_be16(table->VceLevel[count].MinVoltage);
2679 	}
2680 
2681 	return ret;
2682 
2683 }
2684 
ci_populate_smc_acp_level(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2685 static int ci_populate_smc_acp_level(struct radeon_device *rdev,
2686 				     SMU7_Discrete_DpmTable *table)
2687 {
2688 	u32 count;
2689 	struct atom_clock_dividers dividers;
2690 	int ret = -EINVAL;
2691 
2692 	table->AcpLevelCount = (u8)
2693 		(rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count);
2694 
2695 	for (count = 0; count < table->AcpLevelCount; count++) {
2696 		table->AcpLevel[count].Frequency =
2697 			rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].clk;
2698 		table->AcpLevel[count].MinVoltage =
2699 			rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].v;
2700 		table->AcpLevel[count].MinPhases = 1;
2701 
2702 		ret = radeon_atom_get_clock_dividers(rdev,
2703 						     COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2704 						     table->AcpLevel[count].Frequency, false, &dividers);
2705 		if (ret)
2706 			return ret;
2707 
2708 		table->AcpLevel[count].Divider = (u8)dividers.post_divider;
2709 
2710 		table->AcpLevel[count].Frequency = cpu_to_be32(table->AcpLevel[count].Frequency);
2711 		table->AcpLevel[count].MinVoltage = cpu_to_be16(table->AcpLevel[count].MinVoltage);
2712 	}
2713 
2714 	return ret;
2715 }
2716 
ci_populate_smc_samu_level(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2717 static int ci_populate_smc_samu_level(struct radeon_device *rdev,
2718 				      SMU7_Discrete_DpmTable *table)
2719 {
2720 	u32 count;
2721 	struct atom_clock_dividers dividers;
2722 	int ret = -EINVAL;
2723 
2724 	table->SamuLevelCount =
2725 		rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count;
2726 
2727 	for (count = 0; count < table->SamuLevelCount; count++) {
2728 		table->SamuLevel[count].Frequency =
2729 			rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].clk;
2730 		table->SamuLevel[count].MinVoltage =
2731 			rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE;
2732 		table->SamuLevel[count].MinPhases = 1;
2733 
2734 		ret = radeon_atom_get_clock_dividers(rdev,
2735 						     COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2736 						     table->SamuLevel[count].Frequency, false, &dividers);
2737 		if (ret)
2738 			return ret;
2739 
2740 		table->SamuLevel[count].Divider = (u8)dividers.post_divider;
2741 
2742 		table->SamuLevel[count].Frequency = cpu_to_be32(table->SamuLevel[count].Frequency);
2743 		table->SamuLevel[count].MinVoltage = cpu_to_be16(table->SamuLevel[count].MinVoltage);
2744 	}
2745 
2746 	return ret;
2747 }
2748 
ci_calculate_mclk_params(struct radeon_device * rdev,u32 memory_clock,SMU7_Discrete_MemoryLevel * mclk,bool strobe_mode,bool dll_state_on)2749 static int ci_calculate_mclk_params(struct radeon_device *rdev,
2750 				    u32 memory_clock,
2751 				    SMU7_Discrete_MemoryLevel *mclk,
2752 				    bool strobe_mode,
2753 				    bool dll_state_on)
2754 {
2755 	struct ci_power_info *pi = ci_get_pi(rdev);
2756 	u32  dll_cntl = pi->clock_registers.dll_cntl;
2757 	u32  mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl;
2758 	u32  mpll_ad_func_cntl = pi->clock_registers.mpll_ad_func_cntl;
2759 	u32  mpll_dq_func_cntl = pi->clock_registers.mpll_dq_func_cntl;
2760 	u32  mpll_func_cntl = pi->clock_registers.mpll_func_cntl;
2761 	u32  mpll_func_cntl_1 = pi->clock_registers.mpll_func_cntl_1;
2762 	u32  mpll_func_cntl_2 = pi->clock_registers.mpll_func_cntl_2;
2763 	u32  mpll_ss1 = pi->clock_registers.mpll_ss1;
2764 	u32  mpll_ss2 = pi->clock_registers.mpll_ss2;
2765 	struct atom_mpll_param mpll_param;
2766 	int ret;
2767 
2768 	ret = radeon_atom_get_memory_pll_dividers(rdev, memory_clock, strobe_mode, &mpll_param);
2769 	if (ret)
2770 		return ret;
2771 
2772 	mpll_func_cntl &= ~BWCTRL_MASK;
2773 	mpll_func_cntl |= BWCTRL(mpll_param.bwcntl);
2774 
2775 	mpll_func_cntl_1 &= ~(CLKF_MASK | CLKFRAC_MASK | VCO_MODE_MASK);
2776 	mpll_func_cntl_1 |= CLKF(mpll_param.clkf) |
2777 		CLKFRAC(mpll_param.clkfrac) | VCO_MODE(mpll_param.vco_mode);
2778 
2779 	mpll_ad_func_cntl &= ~YCLK_POST_DIV_MASK;
2780 	mpll_ad_func_cntl |= YCLK_POST_DIV(mpll_param.post_div);
2781 
2782 	if (pi->mem_gddr5) {
2783 		mpll_dq_func_cntl &= ~(YCLK_SEL_MASK | YCLK_POST_DIV_MASK);
2784 		mpll_dq_func_cntl |= YCLK_SEL(mpll_param.yclk_sel) |
2785 			YCLK_POST_DIV(mpll_param.post_div);
2786 	}
2787 
2788 	if (pi->caps_mclk_ss_support) {
2789 		struct radeon_atom_ss ss;
2790 		u32 freq_nom;
2791 		u32 tmp;
2792 		u32 reference_clock = rdev->clock.mpll.reference_freq;
2793 
2794 		if (mpll_param.qdr == 1)
2795 			freq_nom = memory_clock * 4 * (1 << mpll_param.post_div);
2796 		else
2797 			freq_nom = memory_clock * 2 * (1 << mpll_param.post_div);
2798 
2799 		tmp = (freq_nom / reference_clock);
2800 		tmp = tmp * tmp;
2801 		if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2802 						     ASIC_INTERNAL_MEMORY_SS, freq_nom)) {
2803 			u32 clks = reference_clock * 5 / ss.rate;
2804 			u32 clkv = (u32)((((131 * ss.percentage * ss.rate) / 100) * tmp) / freq_nom);
2805 
2806 			mpll_ss1 &= ~CLKV_MASK;
2807 			mpll_ss1 |= CLKV(clkv);
2808 
2809 			mpll_ss2 &= ~CLKS_MASK;
2810 			mpll_ss2 |= CLKS(clks);
2811 		}
2812 	}
2813 
2814 	mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2815 	mclk_pwrmgt_cntl |= DLL_SPEED(mpll_param.dll_speed);
2816 
2817 	if (dll_state_on)
2818 		mclk_pwrmgt_cntl |= MRDCK0_PDNB | MRDCK1_PDNB;
2819 	else
2820 		mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB);
2821 
2822 	mclk->MclkFrequency = memory_clock;
2823 	mclk->MpllFuncCntl = mpll_func_cntl;
2824 	mclk->MpllFuncCntl_1 = mpll_func_cntl_1;
2825 	mclk->MpllFuncCntl_2 = mpll_func_cntl_2;
2826 	mclk->MpllAdFuncCntl = mpll_ad_func_cntl;
2827 	mclk->MpllDqFuncCntl = mpll_dq_func_cntl;
2828 	mclk->MclkPwrmgtCntl = mclk_pwrmgt_cntl;
2829 	mclk->DllCntl = dll_cntl;
2830 	mclk->MpllSs1 = mpll_ss1;
2831 	mclk->MpllSs2 = mpll_ss2;
2832 
2833 	return 0;
2834 }
2835 
ci_populate_single_memory_level(struct radeon_device * rdev,u32 memory_clock,SMU7_Discrete_MemoryLevel * memory_level)2836 static int ci_populate_single_memory_level(struct radeon_device *rdev,
2837 					   u32 memory_clock,
2838 					   SMU7_Discrete_MemoryLevel *memory_level)
2839 {
2840 	struct ci_power_info *pi = ci_get_pi(rdev);
2841 	int ret;
2842 	bool dll_state_on;
2843 
2844 	if (rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries) {
2845 		ret = ci_get_dependency_volt_by_clk(rdev,
2846 						    &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
2847 						    memory_clock, &memory_level->MinVddc);
2848 		if (ret)
2849 			return ret;
2850 	}
2851 
2852 	if (rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries) {
2853 		ret = ci_get_dependency_volt_by_clk(rdev,
2854 						    &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
2855 						    memory_clock, &memory_level->MinVddci);
2856 		if (ret)
2857 			return ret;
2858 	}
2859 
2860 	if (rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries) {
2861 		ret = ci_get_dependency_volt_by_clk(rdev,
2862 						    &rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk,
2863 						    memory_clock, &memory_level->MinMvdd);
2864 		if (ret)
2865 			return ret;
2866 	}
2867 
2868 	memory_level->MinVddcPhases = 1;
2869 
2870 	if (pi->vddc_phase_shed_control)
2871 		ci_populate_phase_value_based_on_mclk(rdev,
2872 						      &rdev->pm.dpm.dyn_state.phase_shedding_limits_table,
2873 						      memory_clock,
2874 						      &memory_level->MinVddcPhases);
2875 
2876 	memory_level->EnabledForThrottle = 1;
2877 	memory_level->UpH = 0;
2878 	memory_level->DownH = 100;
2879 	memory_level->VoltageDownH = 0;
2880 	memory_level->ActivityLevel = (u16)pi->mclk_activity_target;
2881 
2882 	memory_level->StutterEnable = false;
2883 	memory_level->StrobeEnable = false;
2884 	memory_level->EdcReadEnable = false;
2885 	memory_level->EdcWriteEnable = false;
2886 	memory_level->RttEnable = false;
2887 
2888 	memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
2889 
2890 	if (pi->mclk_stutter_mode_threshold &&
2891 	    (memory_clock <= pi->mclk_stutter_mode_threshold) &&
2892 	    (pi->uvd_enabled == false) &&
2893 	    (RREG32(DPG_PIPE_STUTTER_CONTROL) & STUTTER_ENABLE) &&
2894 	    (rdev->pm.dpm.new_active_crtc_count <= 2))
2895 		memory_level->StutterEnable = true;
2896 
2897 	if (pi->mclk_strobe_mode_threshold &&
2898 	    (memory_clock <= pi->mclk_strobe_mode_threshold))
2899 		memory_level->StrobeEnable = 1;
2900 
2901 	if (pi->mem_gddr5) {
2902 		memory_level->StrobeRatio =
2903 			si_get_mclk_frequency_ratio(memory_clock, memory_level->StrobeEnable);
2904 		if (pi->mclk_edc_enable_threshold &&
2905 		    (memory_clock > pi->mclk_edc_enable_threshold))
2906 			memory_level->EdcReadEnable = true;
2907 
2908 		if (pi->mclk_edc_wr_enable_threshold &&
2909 		    (memory_clock > pi->mclk_edc_wr_enable_threshold))
2910 			memory_level->EdcWriteEnable = true;
2911 
2912 		if (memory_level->StrobeEnable) {
2913 			if (si_get_mclk_frequency_ratio(memory_clock, true) >=
2914 			    ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2915 				dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2916 			else
2917 				dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2918 		} else {
2919 			dll_state_on = pi->dll_default_on;
2920 		}
2921 	} else {
2922 		memory_level->StrobeRatio = si_get_ddr3_mclk_frequency_ratio(memory_clock);
2923 		dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2924 	}
2925 
2926 	ret = ci_calculate_mclk_params(rdev, memory_clock, memory_level, memory_level->StrobeEnable, dll_state_on);
2927 	if (ret)
2928 		return ret;
2929 
2930 	memory_level->MinVddc = cpu_to_be32(memory_level->MinVddc * VOLTAGE_SCALE);
2931 	memory_level->MinVddcPhases = cpu_to_be32(memory_level->MinVddcPhases);
2932 	memory_level->MinVddci = cpu_to_be32(memory_level->MinVddci * VOLTAGE_SCALE);
2933 	memory_level->MinMvdd = cpu_to_be32(memory_level->MinMvdd * VOLTAGE_SCALE);
2934 
2935 	memory_level->MclkFrequency = cpu_to_be32(memory_level->MclkFrequency);
2936 	memory_level->ActivityLevel = cpu_to_be16(memory_level->ActivityLevel);
2937 	memory_level->MpllFuncCntl = cpu_to_be32(memory_level->MpllFuncCntl);
2938 	memory_level->MpllFuncCntl_1 = cpu_to_be32(memory_level->MpllFuncCntl_1);
2939 	memory_level->MpllFuncCntl_2 = cpu_to_be32(memory_level->MpllFuncCntl_2);
2940 	memory_level->MpllAdFuncCntl = cpu_to_be32(memory_level->MpllAdFuncCntl);
2941 	memory_level->MpllDqFuncCntl = cpu_to_be32(memory_level->MpllDqFuncCntl);
2942 	memory_level->MclkPwrmgtCntl = cpu_to_be32(memory_level->MclkPwrmgtCntl);
2943 	memory_level->DllCntl = cpu_to_be32(memory_level->DllCntl);
2944 	memory_level->MpllSs1 = cpu_to_be32(memory_level->MpllSs1);
2945 	memory_level->MpllSs2 = cpu_to_be32(memory_level->MpllSs2);
2946 
2947 	return 0;
2948 }
2949 
ci_populate_smc_acpi_level(struct radeon_device * rdev,SMU7_Discrete_DpmTable * table)2950 static int ci_populate_smc_acpi_level(struct radeon_device *rdev,
2951 				      SMU7_Discrete_DpmTable *table)
2952 {
2953 	struct ci_power_info *pi = ci_get_pi(rdev);
2954 	struct atom_clock_dividers dividers;
2955 	SMU7_Discrete_VoltageLevel voltage_level;
2956 	u32 spll_func_cntl = pi->clock_registers.cg_spll_func_cntl;
2957 	u32 spll_func_cntl_2 = pi->clock_registers.cg_spll_func_cntl_2;
2958 	u32 dll_cntl = pi->clock_registers.dll_cntl;
2959 	u32 mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl;
2960 	int ret;
2961 
2962 	table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
2963 
2964 	if (pi->acpi_vddc)
2965 		table->ACPILevel.MinVddc = cpu_to_be32(pi->acpi_vddc * VOLTAGE_SCALE);
2966 	else
2967 		table->ACPILevel.MinVddc = cpu_to_be32(pi->min_vddc_in_pp_table * VOLTAGE_SCALE);
2968 
2969 	table->ACPILevel.MinVddcPhases = pi->vddc_phase_shed_control ? 0 : 1;
2970 
2971 	table->ACPILevel.SclkFrequency = rdev->clock.spll.reference_freq;
2972 
2973 	ret = radeon_atom_get_clock_dividers(rdev,
2974 					     COMPUTE_GPUCLK_INPUT_FLAG_SCLK,
2975 					     table->ACPILevel.SclkFrequency, false, &dividers);
2976 	if (ret)
2977 		return ret;
2978 
2979 	table->ACPILevel.SclkDid = (u8)dividers.post_divider;
2980 	table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
2981 	table->ACPILevel.DeepSleepDivId = 0;
2982 
2983 	spll_func_cntl &= ~SPLL_PWRON;
2984 	spll_func_cntl |= SPLL_RESET;
2985 
2986 	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2987 	spll_func_cntl_2 |= SCLK_MUX_SEL(4);
2988 
2989 	table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
2990 	table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
2991 	table->ACPILevel.CgSpllFuncCntl3 = pi->clock_registers.cg_spll_func_cntl_3;
2992 	table->ACPILevel.CgSpllFuncCntl4 = pi->clock_registers.cg_spll_func_cntl_4;
2993 	table->ACPILevel.SpllSpreadSpectrum = pi->clock_registers.cg_spll_spread_spectrum;
2994 	table->ACPILevel.SpllSpreadSpectrum2 = pi->clock_registers.cg_spll_spread_spectrum_2;
2995 	table->ACPILevel.CcPwrDynRm = 0;
2996 	table->ACPILevel.CcPwrDynRm1 = 0;
2997 
2998 	table->ACPILevel.Flags = cpu_to_be32(table->ACPILevel.Flags);
2999 	table->ACPILevel.MinVddcPhases = cpu_to_be32(table->ACPILevel.MinVddcPhases);
3000 	table->ACPILevel.SclkFrequency = cpu_to_be32(table->ACPILevel.SclkFrequency);
3001 	table->ACPILevel.CgSpllFuncCntl = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl);
3002 	table->ACPILevel.CgSpllFuncCntl2 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl2);
3003 	table->ACPILevel.CgSpllFuncCntl3 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl3);
3004 	table->ACPILevel.CgSpllFuncCntl4 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl4);
3005 	table->ACPILevel.SpllSpreadSpectrum = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum);
3006 	table->ACPILevel.SpllSpreadSpectrum2 = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum2);
3007 	table->ACPILevel.CcPwrDynRm = cpu_to_be32(table->ACPILevel.CcPwrDynRm);
3008 	table->ACPILevel.CcPwrDynRm1 = cpu_to_be32(table->ACPILevel.CcPwrDynRm1);
3009 
3010 	table->MemoryACPILevel.MinVddc = table->ACPILevel.MinVddc;
3011 	table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases;
3012 
3013 	if (pi->vddci_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
3014 		if (pi->acpi_vddci)
3015 			table->MemoryACPILevel.MinVddci =
3016 				cpu_to_be32(pi->acpi_vddci * VOLTAGE_SCALE);
3017 		else
3018 			table->MemoryACPILevel.MinVddci =
3019 				cpu_to_be32(pi->min_vddci_in_pp_table * VOLTAGE_SCALE);
3020 	}
3021 
3022 	if (ci_populate_mvdd_value(rdev, 0, &voltage_level))
3023 		table->MemoryACPILevel.MinMvdd = 0;
3024 	else
3025 		table->MemoryACPILevel.MinMvdd =
3026 			cpu_to_be32(voltage_level.Voltage * VOLTAGE_SCALE);
3027 
3028 	mclk_pwrmgt_cntl |= MRDCK0_RESET | MRDCK1_RESET;
3029 	mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB);
3030 
3031 	dll_cntl &= ~(MRDCK0_BYPASS | MRDCK1_BYPASS);
3032 
3033 	table->MemoryACPILevel.DllCntl = cpu_to_be32(dll_cntl);
3034 	table->MemoryACPILevel.MclkPwrmgtCntl = cpu_to_be32(mclk_pwrmgt_cntl);
3035 	table->MemoryACPILevel.MpllAdFuncCntl =
3036 		cpu_to_be32(pi->clock_registers.mpll_ad_func_cntl);
3037 	table->MemoryACPILevel.MpllDqFuncCntl =
3038 		cpu_to_be32(pi->clock_registers.mpll_dq_func_cntl);
3039 	table->MemoryACPILevel.MpllFuncCntl =
3040 		cpu_to_be32(pi->clock_registers.mpll_func_cntl);
3041 	table->MemoryACPILevel.MpllFuncCntl_1 =
3042 		cpu_to_be32(pi->clock_registers.mpll_func_cntl_1);
3043 	table->MemoryACPILevel.MpllFuncCntl_2 =
3044 		cpu_to_be32(pi->clock_registers.mpll_func_cntl_2);
3045 	table->MemoryACPILevel.MpllSs1 = cpu_to_be32(pi->clock_registers.mpll_ss1);
3046 	table->MemoryACPILevel.MpllSs2 = cpu_to_be32(pi->clock_registers.mpll_ss2);
3047 
3048 	table->MemoryACPILevel.EnabledForThrottle = 0;
3049 	table->MemoryACPILevel.EnabledForActivity = 0;
3050 	table->MemoryACPILevel.UpH = 0;
3051 	table->MemoryACPILevel.DownH = 100;
3052 	table->MemoryACPILevel.VoltageDownH = 0;
3053 	table->MemoryACPILevel.ActivityLevel =
3054 		cpu_to_be16((u16)pi->mclk_activity_target);
3055 
3056 	table->MemoryACPILevel.StutterEnable = false;
3057 	table->MemoryACPILevel.StrobeEnable = false;
3058 	table->MemoryACPILevel.EdcReadEnable = false;
3059 	table->MemoryACPILevel.EdcWriteEnable = false;
3060 	table->MemoryACPILevel.RttEnable = false;
3061 
3062 	return 0;
3063 }
3064 
3065 
ci_enable_ulv(struct radeon_device * rdev,bool enable)3066 static int ci_enable_ulv(struct radeon_device *rdev, bool enable)
3067 {
3068 	struct ci_power_info *pi = ci_get_pi(rdev);
3069 	struct ci_ulv_parm *ulv = &pi->ulv;
3070 
3071 	if (ulv->supported) {
3072 		if (enable)
3073 			return (ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableULV) == PPSMC_Result_OK) ?
3074 				0 : -EINVAL;
3075 		else
3076 			return (ci_send_msg_to_smc(rdev, PPSMC_MSG_DisableULV) == PPSMC_Result_OK) ?
3077 				0 : -EINVAL;
3078 	}
3079 
3080 	return 0;
3081 }
3082 
ci_populate_ulv_level(struct radeon_device * rdev,SMU7_Discrete_Ulv * state)3083 static int ci_populate_ulv_level(struct radeon_device *rdev,
3084 				 SMU7_Discrete_Ulv *state)
3085 {
3086 	struct ci_power_info *pi = ci_get_pi(rdev);
3087 	u16 ulv_voltage = rdev->pm.dpm.backbias_response_time;
3088 
3089 	state->CcPwrDynRm = 0;
3090 	state->CcPwrDynRm1 = 0;
3091 
3092 	if (ulv_voltage == 0) {
3093 		pi->ulv.supported = false;
3094 		return 0;
3095 	}
3096 
3097 	if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
3098 		if (ulv_voltage > rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v)
3099 			state->VddcOffset = 0;
3100 		else
3101 			state->VddcOffset =
3102 				rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage;
3103 	} else {
3104 		if (ulv_voltage > rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v)
3105 			state->VddcOffsetVid = 0;
3106 		else
3107 			state->VddcOffsetVid = (u8)
3108 				((rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage) *
3109 				 VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1);
3110 	}
3111 	state->VddcPhase = pi->vddc_phase_shed_control ? 0 : 1;
3112 
3113 	state->CcPwrDynRm = cpu_to_be32(state->CcPwrDynRm);
3114 	state->CcPwrDynRm1 = cpu_to_be32(state->CcPwrDynRm1);
3115 	state->VddcOffset = cpu_to_be16(state->VddcOffset);
3116 
3117 	return 0;
3118 }
3119 
ci_calculate_sclk_params(struct radeon_device * rdev,u32 engine_clock,SMU7_Discrete_GraphicsLevel * sclk)3120 static int ci_calculate_sclk_params(struct radeon_device *rdev,
3121 				    u32 engine_clock,
3122 				    SMU7_Discrete_GraphicsLevel *sclk)
3123 {
3124 	struct ci_power_info *pi = ci_get_pi(rdev);
3125 	struct atom_clock_dividers dividers;
3126 	u32 spll_func_cntl_3 = pi->clock_registers.cg_spll_func_cntl_3;
3127 	u32 spll_func_cntl_4 = pi->clock_registers.cg_spll_func_cntl_4;
3128 	u32 cg_spll_spread_spectrum = pi->clock_registers.cg_spll_spread_spectrum;
3129 	u32 cg_spll_spread_spectrum_2 = pi->clock_registers.cg_spll_spread_spectrum_2;
3130 	u32 reference_clock = rdev->clock.spll.reference_freq;
3131 	u32 reference_divider;
3132 	u32 fbdiv;
3133 	int ret;
3134 
3135 	ret = radeon_atom_get_clock_dividers(rdev,
3136 					     COMPUTE_GPUCLK_INPUT_FLAG_SCLK,
3137 					     engine_clock, false, &dividers);
3138 	if (ret)
3139 		return ret;
3140 
3141 	reference_divider = 1 + dividers.ref_div;
3142 	fbdiv = dividers.fb_div & 0x3FFFFFF;
3143 
3144 	spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
3145 	spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
3146 	spll_func_cntl_3 |= SPLL_DITHEN;
3147 
3148 	if (pi->caps_sclk_ss_support) {
3149 		struct radeon_atom_ss ss;
3150 		u32 vco_freq = engine_clock * dividers.post_div;
3151 
3152 		if (radeon_atombios_get_asic_ss_info(rdev, &ss,
3153 						     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
3154 			u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
3155 			u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
3156 
3157 			cg_spll_spread_spectrum &= ~CLK_S_MASK;
3158 			cg_spll_spread_spectrum |= CLK_S(clk_s);
3159 			cg_spll_spread_spectrum |= SSEN;
3160 
3161 			cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
3162 			cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
3163 		}
3164 	}
3165 
3166 	sclk->SclkFrequency = engine_clock;
3167 	sclk->CgSpllFuncCntl3 = spll_func_cntl_3;
3168 	sclk->CgSpllFuncCntl4 = spll_func_cntl_4;
3169 	sclk->SpllSpreadSpectrum = cg_spll_spread_spectrum;
3170 	sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
3171 	sclk->SclkDid = (u8)dividers.post_divider;
3172 
3173 	return 0;
3174 }
3175 
ci_populate_single_graphic_level(struct radeon_device * rdev,u32 engine_clock,u16 sclk_activity_level_t,SMU7_Discrete_GraphicsLevel * graphic_level)3176 static int ci_populate_single_graphic_level(struct radeon_device *rdev,
3177 					    u32 engine_clock,
3178 					    u16 sclk_activity_level_t,
3179 					    SMU7_Discrete_GraphicsLevel *graphic_level)
3180 {
3181 	struct ci_power_info *pi = ci_get_pi(rdev);
3182 	int ret;
3183 
3184 	ret = ci_calculate_sclk_params(rdev, engine_clock, graphic_level);
3185 	if (ret)
3186 		return ret;
3187 
3188 	ret = ci_get_dependency_volt_by_clk(rdev,
3189 					    &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
3190 					    engine_clock, &graphic_level->MinVddc);
3191 	if (ret)
3192 		return ret;
3193 
3194 	graphic_level->SclkFrequency = engine_clock;
3195 
3196 	graphic_level->Flags =  0;
3197 	graphic_level->MinVddcPhases = 1;
3198 
3199 	if (pi->vddc_phase_shed_control)
3200 		ci_populate_phase_value_based_on_sclk(rdev,
3201 						      &rdev->pm.dpm.dyn_state.phase_shedding_limits_table,
3202 						      engine_clock,
3203 						      &graphic_level->MinVddcPhases);
3204 
3205 	graphic_level->ActivityLevel = sclk_activity_level_t;
3206 
3207 	graphic_level->CcPwrDynRm = 0;
3208 	graphic_level->CcPwrDynRm1 = 0;
3209 	graphic_level->EnabledForThrottle = 1;
3210 	graphic_level->UpH = 0;
3211 	graphic_level->DownH = 0;
3212 	graphic_level->VoltageDownH = 0;
3213 	graphic_level->PowerThrottle = 0;
3214 
3215 	if (pi->caps_sclk_ds)
3216 		graphic_level->DeepSleepDivId = ci_get_sleep_divider_id_from_clock(rdev,
3217 										   engine_clock,
3218 										   CISLAND_MINIMUM_ENGINE_CLOCK);
3219 
3220 	graphic_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
3221 
3222 	graphic_level->Flags = cpu_to_be32(graphic_level->Flags);
3223 	graphic_level->MinVddc = cpu_to_be32(graphic_level->MinVddc * VOLTAGE_SCALE);
3224 	graphic_level->MinVddcPhases = cpu_to_be32(graphic_level->MinVddcPhases);
3225 	graphic_level->SclkFrequency = cpu_to_be32(graphic_level->SclkFrequency);
3226 	graphic_level->ActivityLevel = cpu_to_be16(graphic_level->ActivityLevel);
3227 	graphic_level->CgSpllFuncCntl3 = cpu_to_be32(graphic_level->CgSpllFuncCntl3);
3228 	graphic_level->CgSpllFuncCntl4 = cpu_to_be32(graphic_level->CgSpllFuncCntl4);
3229 	graphic_level->SpllSpreadSpectrum = cpu_to_be32(graphic_level->SpllSpreadSpectrum);
3230 	graphic_level->SpllSpreadSpectrum2 = cpu_to_be32(graphic_level->SpllSpreadSpectrum2);
3231 	graphic_level->CcPwrDynRm = cpu_to_be32(graphic_level->CcPwrDynRm);
3232 	graphic_level->CcPwrDynRm1 = cpu_to_be32(graphic_level->CcPwrDynRm1);
3233 
3234 	return 0;
3235 }
3236 
ci_populate_all_graphic_levels(struct radeon_device * rdev)3237 static int ci_populate_all_graphic_levels(struct radeon_device *rdev)
3238 {
3239 	struct ci_power_info *pi = ci_get_pi(rdev);
3240 	struct ci_dpm_table *dpm_table = &pi->dpm_table;
3241 	u32 level_array_address = pi->dpm_table_start +
3242 		offsetof(SMU7_Discrete_DpmTable, GraphicsLevel);
3243 	u32 level_array_size = sizeof(SMU7_Discrete_GraphicsLevel) *
3244 		SMU7_MAX_LEVELS_GRAPHICS;
3245 	SMU7_Discrete_GraphicsLevel *levels = pi->smc_state_table.GraphicsLevel;
3246 	u32 i, ret;
3247 
3248 	memset(levels, 0, level_array_size);
3249 
3250 	for (i = 0; i < dpm_table->sclk_table.count; i++) {
3251 		ret = ci_populate_single_graphic_level(rdev,
3252 						       dpm_table->sclk_table.dpm_levels[i].value,
3253 						       (u16)pi->activity_target[i],
3254 						       &pi->smc_state_table.GraphicsLevel[i]);
3255 		if (ret)
3256 			return ret;
3257 		if (i > 1)
3258 			pi->smc_state_table.GraphicsLevel[i].DeepSleepDivId = 0;
3259 		if (i == (dpm_table->sclk_table.count - 1))
3260 			pi->smc_state_table.GraphicsLevel[i].DisplayWatermark =
3261 				PPSMC_DISPLAY_WATERMARK_HIGH;
3262 	}
3263 	pi->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1;
3264 
3265 	pi->smc_state_table.GraphicsDpmLevelCount = (u8)dpm_table->sclk_table.count;
3266 	pi->dpm_level_enable_mask.sclk_dpm_enable_mask =
3267 		ci_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
3268 
3269 	ret = ci_copy_bytes_to_smc(rdev, level_array_address,
3270 				   (u8 *)levels, level_array_size,
3271 				   pi->sram_end);
3272 	if (ret)
3273 		return ret;
3274 
3275 	return 0;
3276 }
3277 
ci_populate_ulv_state(struct radeon_device * rdev,SMU7_Discrete_Ulv * ulv_level)3278 static int ci_populate_ulv_state(struct radeon_device *rdev,
3279 				 SMU7_Discrete_Ulv *ulv_level)
3280 {
3281 	return ci_populate_ulv_level(rdev, ulv_level);
3282 }
3283 
ci_populate_all_memory_levels(struct radeon_device * rdev)3284 static int ci_populate_all_memory_levels(struct radeon_device *rdev)
3285 {
3286 	struct ci_power_info *pi = ci_get_pi(rdev);
3287 	struct ci_dpm_table *dpm_table = &pi->dpm_table;
3288 	u32 level_array_address = pi->dpm_table_start +
3289 		offsetof(SMU7_Discrete_DpmTable, MemoryLevel);
3290 	u32 level_array_size = sizeof(SMU7_Discrete_MemoryLevel) *
3291 		SMU7_MAX_LEVELS_MEMORY;
3292 	SMU7_Discrete_MemoryLevel *levels = pi->smc_state_table.MemoryLevel;
3293 	u32 i, ret;
3294 
3295 	memset(levels, 0, level_array_size);
3296 
3297 	for (i = 0; i < dpm_table->mclk_table.count; i++) {
3298 		if (dpm_table->mclk_table.dpm_levels[i].value == 0)
3299 			return -EINVAL;
3300 		ret = ci_populate_single_memory_level(rdev,
3301 						      dpm_table->mclk_table.dpm_levels[i].value,
3302 						      &pi->smc_state_table.MemoryLevel[i]);
3303 		if (ret)
3304 			return ret;
3305 	}
3306 
3307 	pi->smc_state_table.MemoryLevel[0].EnabledForActivity = 1;
3308 
3309 	if ((dpm_table->mclk_table.count >= 2) &&
3310 	    ((rdev->pdev->device == 0x67B0) || (rdev->pdev->device == 0x67B1))) {
3311 		pi->smc_state_table.MemoryLevel[1].MinVddc =
3312 			pi->smc_state_table.MemoryLevel[0].MinVddc;
3313 		pi->smc_state_table.MemoryLevel[1].MinVddcPhases =
3314 			pi->smc_state_table.MemoryLevel[0].MinVddcPhases;
3315 	}
3316 
3317 	pi->smc_state_table.MemoryLevel[0].ActivityLevel = cpu_to_be16(0x1F);
3318 
3319 	pi->smc_state_table.MemoryDpmLevelCount = (u8)dpm_table->mclk_table.count;
3320 	pi->dpm_level_enable_mask.mclk_dpm_enable_mask =
3321 		ci_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
3322 
3323 	pi->smc_state_table.MemoryLevel[dpm_table->mclk_table.count - 1].DisplayWatermark =
3324 		PPSMC_DISPLAY_WATERMARK_HIGH;
3325 
3326 	ret = ci_copy_bytes_to_smc(rdev, level_array_address,
3327 				   (u8 *)levels, level_array_size,
3328 				   pi->sram_end);
3329 	if (ret)
3330 		return ret;
3331 
3332 	return 0;
3333 }
3334 
ci_reset_single_dpm_table(struct radeon_device * rdev,struct ci_single_dpm_table * dpm_table,u32 count)3335 static void ci_reset_single_dpm_table(struct radeon_device *rdev,
3336 				      struct ci_single_dpm_table* dpm_table,
3337 				      u32 count)
3338 {
3339 	u32 i;
3340 
3341 	dpm_table->count = count;
3342 	for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++)
3343 		dpm_table->dpm_levels[i].enabled = false;
3344 }
3345 
ci_setup_pcie_table_entry(struct ci_single_dpm_table * dpm_table,u32 index,u32 pcie_gen,u32 pcie_lanes)3346 static void ci_setup_pcie_table_entry(struct ci_single_dpm_table* dpm_table,
3347 				      u32 index, u32 pcie_gen, u32 pcie_lanes)
3348 {
3349 	dpm_table->dpm_levels[index].value = pcie_gen;
3350 	dpm_table->dpm_levels[index].param1 = pcie_lanes;
3351 	dpm_table->dpm_levels[index].enabled = true;
3352 }
3353 
ci_setup_default_pcie_tables(struct radeon_device * rdev)3354 static int ci_setup_default_pcie_tables(struct radeon_device *rdev)
3355 {
3356 	struct ci_power_info *pi = ci_get_pi(rdev);
3357 
3358 	if (!pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels)
3359 		return -EINVAL;
3360 
3361 	if (pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels) {
3362 		pi->pcie_gen_powersaving = pi->pcie_gen_performance;
3363 		pi->pcie_lane_powersaving = pi->pcie_lane_performance;
3364 	} else if (!pi->use_pcie_performance_levels && pi->use_pcie_powersaving_levels) {
3365 		pi->pcie_gen_performance = pi->pcie_gen_powersaving;
3366 		pi->pcie_lane_performance = pi->pcie_lane_powersaving;
3367 	}
3368 
3369 	ci_reset_single_dpm_table(rdev,
3370 				  &pi->dpm_table.pcie_speed_table,
3371 				  SMU7_MAX_LEVELS_LINK);
3372 
3373 	if (rdev->family == CHIP_BONAIRE)
3374 		ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0,
3375 					  pi->pcie_gen_powersaving.min,
3376 					  pi->pcie_lane_powersaving.max);
3377 	else
3378 		ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0,
3379 					  pi->pcie_gen_powersaving.min,
3380 					  pi->pcie_lane_powersaving.min);
3381 	ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 1,
3382 				  pi->pcie_gen_performance.min,
3383 				  pi->pcie_lane_performance.min);
3384 	ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 2,
3385 				  pi->pcie_gen_powersaving.min,
3386 				  pi->pcie_lane_powersaving.max);
3387 	ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 3,
3388 				  pi->pcie_gen_performance.min,
3389 				  pi->pcie_lane_performance.max);
3390 	ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 4,
3391 				  pi->pcie_gen_powersaving.max,
3392 				  pi->pcie_lane_powersaving.max);
3393 	ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 5,
3394 				  pi->pcie_gen_performance.max,
3395 				  pi->pcie_lane_performance.max);
3396 
3397 	pi->dpm_table.pcie_speed_table.count = 6;
3398 
3399 	return 0;
3400 }
3401 
ci_setup_default_dpm_tables(struct radeon_device * rdev)3402 static int ci_setup_default_dpm_tables(struct radeon_device *rdev)
3403 {
3404 	struct ci_power_info *pi = ci_get_pi(rdev);
3405 	struct radeon_clock_voltage_dependency_table *allowed_sclk_vddc_table =
3406 		&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
3407 	struct radeon_clock_voltage_dependency_table *allowed_mclk_table =
3408 		&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk;
3409 	struct radeon_cac_leakage_table *std_voltage_table =
3410 		&rdev->pm.dpm.dyn_state.cac_leakage_table;
3411 	u32 i;
3412 
3413 	if (allowed_sclk_vddc_table == NULL)
3414 		return -EINVAL;
3415 	if (allowed_sclk_vddc_table->count < 1)
3416 		return -EINVAL;
3417 	if (allowed_mclk_table == NULL)
3418 		return -EINVAL;
3419 	if (allowed_mclk_table->count < 1)
3420 		return -EINVAL;
3421 
3422 	memset(&pi->dpm_table, 0, sizeof(struct ci_dpm_table));
3423 
3424 	ci_reset_single_dpm_table(rdev,
3425 				  &pi->dpm_table.sclk_table,
3426 				  SMU7_MAX_LEVELS_GRAPHICS);
3427 	ci_reset_single_dpm_table(rdev,
3428 				  &pi->dpm_table.mclk_table,
3429 				  SMU7_MAX_LEVELS_MEMORY);
3430 	ci_reset_single_dpm_table(rdev,
3431 				  &pi->dpm_table.vddc_table,
3432 				  SMU7_MAX_LEVELS_VDDC);
3433 	ci_reset_single_dpm_table(rdev,
3434 				  &pi->dpm_table.vddci_table,
3435 				  SMU7_MAX_LEVELS_VDDCI);
3436 	ci_reset_single_dpm_table(rdev,
3437 				  &pi->dpm_table.mvdd_table,
3438 				  SMU7_MAX_LEVELS_MVDD);
3439 
3440 	pi->dpm_table.sclk_table.count = 0;
3441 	for (i = 0; i < allowed_sclk_vddc_table->count; i++) {
3442 		if ((i == 0) ||
3443 		    (pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count-1].value !=
3444 		     allowed_sclk_vddc_table->entries[i].clk)) {
3445 			pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].value =
3446 				allowed_sclk_vddc_table->entries[i].clk;
3447 			pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].enabled =
3448 				(i == 0) ? true : false;
3449 			pi->dpm_table.sclk_table.count++;
3450 		}
3451 	}
3452 
3453 	pi->dpm_table.mclk_table.count = 0;
3454 	for (i = 0; i < allowed_mclk_table->count; i++) {
3455 		if ((i == 0) ||
3456 		    (pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count-1].value !=
3457 		     allowed_mclk_table->entries[i].clk)) {
3458 			pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].value =
3459 				allowed_mclk_table->entries[i].clk;
3460 			pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].enabled =
3461 				(i == 0) ? true : false;
3462 			pi->dpm_table.mclk_table.count++;
3463 		}
3464 	}
3465 
3466 	for (i = 0; i < allowed_sclk_vddc_table->count; i++) {
3467 		pi->dpm_table.vddc_table.dpm_levels[i].value =
3468 			allowed_sclk_vddc_table->entries[i].v;
3469 		pi->dpm_table.vddc_table.dpm_levels[i].param1 =
3470 			std_voltage_table->entries[i].leakage;
3471 		pi->dpm_table.vddc_table.dpm_levels[i].enabled = true;
3472 	}
3473 	pi->dpm_table.vddc_table.count = allowed_sclk_vddc_table->count;
3474 
3475 	allowed_mclk_table = &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk;
3476 	if (allowed_mclk_table) {
3477 		for (i = 0; i < allowed_mclk_table->count; i++) {
3478 			pi->dpm_table.vddci_table.dpm_levels[i].value =
3479 				allowed_mclk_table->entries[i].v;
3480 			pi->dpm_table.vddci_table.dpm_levels[i].enabled = true;
3481 		}
3482 		pi->dpm_table.vddci_table.count = allowed_mclk_table->count;
3483 	}
3484 
3485 	allowed_mclk_table = &rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk;
3486 	if (allowed_mclk_table) {
3487 		for (i = 0; i < allowed_mclk_table->count; i++) {
3488 			pi->dpm_table.mvdd_table.dpm_levels[i].value =
3489 				allowed_mclk_table->entries[i].v;
3490 			pi->dpm_table.mvdd_table.dpm_levels[i].enabled = true;
3491 		}
3492 		pi->dpm_table.mvdd_table.count = allowed_mclk_table->count;
3493 	}
3494 
3495 	ci_setup_default_pcie_tables(rdev);
3496 
3497 	return 0;
3498 }
3499 
ci_find_boot_level(struct ci_single_dpm_table * table,u32 value,u32 * boot_level)3500 static int ci_find_boot_level(struct ci_single_dpm_table *table,
3501 			      u32 value, u32 *boot_level)
3502 {
3503 	u32 i;
3504 	int ret = -EINVAL;
3505 
3506 	for(i = 0; i < table->count; i++) {
3507 		if (value == table->dpm_levels[i].value) {
3508 			*boot_level = i;
3509 			ret = 0;
3510 		}
3511 	}
3512 
3513 	return ret;
3514 }
3515 
ci_init_smc_table(struct radeon_device * rdev)3516 static int ci_init_smc_table(struct radeon_device *rdev)
3517 {
3518 	struct ci_power_info *pi = ci_get_pi(rdev);
3519 	struct ci_ulv_parm *ulv = &pi->ulv;
3520 	struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
3521 	SMU7_Discrete_DpmTable *table = &pi->smc_state_table;
3522 	int ret;
3523 
3524 	ret = ci_setup_default_dpm_tables(rdev);
3525 	if (ret)
3526 		return ret;
3527 
3528 	if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE)
3529 		ci_populate_smc_voltage_tables(rdev, table);
3530 
3531 	ci_init_fps_limits(rdev);
3532 
3533 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
3534 		table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
3535 
3536 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
3537 		table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
3538 
3539 	if (pi->mem_gddr5)
3540 		table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
3541 
3542 	if (ulv->supported) {
3543 		ret = ci_populate_ulv_state(rdev, &pi->smc_state_table.Ulv);
3544 		if (ret)
3545 			return ret;
3546 		WREG32_SMC(CG_ULV_PARAMETER, ulv->cg_ulv_parameter);
3547 	}
3548 
3549 	ret = ci_populate_all_graphic_levels(rdev);
3550 	if (ret)
3551 		return ret;
3552 
3553 	ret = ci_populate_all_memory_levels(rdev);
3554 	if (ret)
3555 		return ret;
3556 
3557 	ci_populate_smc_link_level(rdev, table);
3558 
3559 	ret = ci_populate_smc_acpi_level(rdev, table);
3560 	if (ret)
3561 		return ret;
3562 
3563 	ret = ci_populate_smc_vce_level(rdev, table);
3564 	if (ret)
3565 		return ret;
3566 
3567 	ret = ci_populate_smc_acp_level(rdev, table);
3568 	if (ret)
3569 		return ret;
3570 
3571 	ret = ci_populate_smc_samu_level(rdev, table);
3572 	if (ret)
3573 		return ret;
3574 
3575 	ret = ci_do_program_memory_timing_parameters(rdev);
3576 	if (ret)
3577 		return ret;
3578 
3579 	ret = ci_populate_smc_uvd_level(rdev, table);
3580 	if (ret)
3581 		return ret;
3582 
3583 	table->UvdBootLevel  = 0;
3584 	table->VceBootLevel  = 0;
3585 	table->AcpBootLevel  = 0;
3586 	table->SamuBootLevel  = 0;
3587 	table->GraphicsBootLevel  = 0;
3588 	table->MemoryBootLevel  = 0;
3589 
3590 	ret = ci_find_boot_level(&pi->dpm_table.sclk_table,
3591 				 pi->vbios_boot_state.sclk_bootup_value,
3592 				 (u32 *)&pi->smc_state_table.GraphicsBootLevel);
3593 
3594 	ret = ci_find_boot_level(&pi->dpm_table.mclk_table,
3595 				 pi->vbios_boot_state.mclk_bootup_value,
3596 				 (u32 *)&pi->smc_state_table.MemoryBootLevel);
3597 
3598 	table->BootVddc = pi->vbios_boot_state.vddc_bootup_value;
3599 	table->BootVddci = pi->vbios_boot_state.vddci_bootup_value;
3600 	table->BootMVdd = pi->vbios_boot_state.mvdd_bootup_value;
3601 
3602 	ci_populate_smc_initial_state(rdev, radeon_boot_state);
3603 
3604 	ret = ci_populate_bapm_parameters_in_dpm_table(rdev);
3605 	if (ret)
3606 		return ret;
3607 
3608 	table->UVDInterval = 1;
3609 	table->VCEInterval = 1;
3610 	table->ACPInterval = 1;
3611 	table->SAMUInterval = 1;
3612 	table->GraphicsVoltageChangeEnable = 1;
3613 	table->GraphicsThermThrottleEnable = 1;
3614 	table->GraphicsInterval = 1;
3615 	table->VoltageInterval = 1;
3616 	table->ThermalInterval = 1;
3617 	table->TemperatureLimitHigh = (u16)((pi->thermal_temp_setting.temperature_high *
3618 					     CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000);
3619 	table->TemperatureLimitLow = (u16)((pi->thermal_temp_setting.temperature_low *
3620 					    CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000);
3621 	table->MemoryVoltageChangeEnable = 1;
3622 	table->MemoryInterval = 1;
3623 	table->VoltageResponseTime = 0;
3624 	table->VddcVddciDelta = 4000;
3625 	table->PhaseResponseTime = 0;
3626 	table->MemoryThermThrottleEnable = 1;
3627 	table->PCIeBootLinkLevel = pi->dpm_table.pcie_speed_table.count - 1;
3628 	table->PCIeGenInterval = 1;
3629 	if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2)
3630 		table->SVI2Enable  = 1;
3631 	else
3632 		table->SVI2Enable  = 0;
3633 
3634 	table->ThermGpio = 17;
3635 	table->SclkStepSize = 0x4000;
3636 
3637 	table->SystemFlags = cpu_to_be32(table->SystemFlags);
3638 	table->SmioMaskVddcVid = cpu_to_be32(table->SmioMaskVddcVid);
3639 	table->SmioMaskVddcPhase = cpu_to_be32(table->SmioMaskVddcPhase);
3640 	table->SmioMaskVddciVid = cpu_to_be32(table->SmioMaskVddciVid);
3641 	table->SmioMaskMvddVid = cpu_to_be32(table->SmioMaskMvddVid);
3642 	table->SclkStepSize = cpu_to_be32(table->SclkStepSize);
3643 	table->TemperatureLimitHigh = cpu_to_be16(table->TemperatureLimitHigh);
3644 	table->TemperatureLimitLow = cpu_to_be16(table->TemperatureLimitLow);
3645 	table->VddcVddciDelta = cpu_to_be16(table->VddcVddciDelta);
3646 	table->VoltageResponseTime = cpu_to_be16(table->VoltageResponseTime);
3647 	table->PhaseResponseTime = cpu_to_be16(table->PhaseResponseTime);
3648 	table->BootVddc = cpu_to_be16(table->BootVddc * VOLTAGE_SCALE);
3649 	table->BootVddci = cpu_to_be16(table->BootVddci * VOLTAGE_SCALE);
3650 	table->BootMVdd = cpu_to_be16(table->BootMVdd * VOLTAGE_SCALE);
3651 
3652 	ret = ci_copy_bytes_to_smc(rdev,
3653 				   pi->dpm_table_start +
3654 				   offsetof(SMU7_Discrete_DpmTable, SystemFlags),
3655 				   (u8 *)&table->SystemFlags,
3656 				   sizeof(SMU7_Discrete_DpmTable) - 3 * sizeof(SMU7_PIDController),
3657 				   pi->sram_end);
3658 	if (ret)
3659 		return ret;
3660 
3661 	return 0;
3662 }
3663 
ci_trim_single_dpm_states(struct radeon_device * rdev,struct ci_single_dpm_table * dpm_table,u32 low_limit,u32 high_limit)3664 static void ci_trim_single_dpm_states(struct radeon_device *rdev,
3665 				      struct ci_single_dpm_table *dpm_table,
3666 				      u32 low_limit, u32 high_limit)
3667 {
3668 	u32 i;
3669 
3670 	for (i = 0; i < dpm_table->count; i++) {
3671 		if ((dpm_table->dpm_levels[i].value < low_limit) ||
3672 		    (dpm_table->dpm_levels[i].value > high_limit))
3673 			dpm_table->dpm_levels[i].enabled = false;
3674 		else
3675 			dpm_table->dpm_levels[i].enabled = true;
3676 	}
3677 }
3678 
ci_trim_pcie_dpm_states(struct radeon_device * rdev,u32 speed_low,u32 lanes_low,u32 speed_high,u32 lanes_high)3679 static void ci_trim_pcie_dpm_states(struct radeon_device *rdev,
3680 				    u32 speed_low, u32 lanes_low,
3681 				    u32 speed_high, u32 lanes_high)
3682 {
3683 	struct ci_power_info *pi = ci_get_pi(rdev);
3684 	struct ci_single_dpm_table *pcie_table = &pi->dpm_table.pcie_speed_table;
3685 	u32 i, j;
3686 
3687 	for (i = 0; i < pcie_table->count; i++) {
3688 		if ((pcie_table->dpm_levels[i].value < speed_low) ||
3689 		    (pcie_table->dpm_levels[i].param1 < lanes_low) ||
3690 		    (pcie_table->dpm_levels[i].value > speed_high) ||
3691 		    (pcie_table->dpm_levels[i].param1 > lanes_high))
3692 			pcie_table->dpm_levels[i].enabled = false;
3693 		else
3694 			pcie_table->dpm_levels[i].enabled = true;
3695 	}
3696 
3697 	for (i = 0; i < pcie_table->count; i++) {
3698 		if (pcie_table->dpm_levels[i].enabled) {
3699 			for (j = i + 1; j < pcie_table->count; j++) {
3700 				if (pcie_table->dpm_levels[j].enabled) {
3701 					if ((pcie_table->dpm_levels[i].value == pcie_table->dpm_levels[j].value) &&
3702 					    (pcie_table->dpm_levels[i].param1 == pcie_table->dpm_levels[j].param1))
3703 						pcie_table->dpm_levels[j].enabled = false;
3704 				}
3705 			}
3706 		}
3707 	}
3708 }
3709 
ci_trim_dpm_states(struct radeon_device * rdev,struct radeon_ps * radeon_state)3710 static int ci_trim_dpm_states(struct radeon_device *rdev,
3711 			      struct radeon_ps *radeon_state)
3712 {
3713 	struct ci_ps *state = ci_get_ps(radeon_state);
3714 	struct ci_power_info *pi = ci_get_pi(rdev);
3715 	u32 high_limit_count;
3716 
3717 	if (state->performance_level_count < 1)
3718 		return -EINVAL;
3719 
3720 	if (state->performance_level_count == 1)
3721 		high_limit_count = 0;
3722 	else
3723 		high_limit_count = 1;
3724 
3725 	ci_trim_single_dpm_states(rdev,
3726 				  &pi->dpm_table.sclk_table,
3727 				  state->performance_levels[0].sclk,
3728 				  state->performance_levels[high_limit_count].sclk);
3729 
3730 	ci_trim_single_dpm_states(rdev,
3731 				  &pi->dpm_table.mclk_table,
3732 				  state->performance_levels[0].mclk,
3733 				  state->performance_levels[high_limit_count].mclk);
3734 
3735 	ci_trim_pcie_dpm_states(rdev,
3736 				state->performance_levels[0].pcie_gen,
3737 				state->performance_levels[0].pcie_lane,
3738 				state->performance_levels[high_limit_count].pcie_gen,
3739 				state->performance_levels[high_limit_count].pcie_lane);
3740 
3741 	return 0;
3742 }
3743 
ci_apply_disp_minimum_voltage_request(struct radeon_device * rdev)3744 static int ci_apply_disp_minimum_voltage_request(struct radeon_device *rdev)
3745 {
3746 	struct radeon_clock_voltage_dependency_table *disp_voltage_table =
3747 		&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk;
3748 	struct radeon_clock_voltage_dependency_table *vddc_table =
3749 		&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
3750 	u32 requested_voltage = 0;
3751 	u32 i;
3752 
3753 	if (disp_voltage_table == NULL)
3754 		return -EINVAL;
3755 	if (!disp_voltage_table->count)
3756 		return -EINVAL;
3757 
3758 	for (i = 0; i < disp_voltage_table->count; i++) {
3759 		if (rdev->clock.current_dispclk == disp_voltage_table->entries[i].clk)
3760 			requested_voltage = disp_voltage_table->entries[i].v;
3761 	}
3762 
3763 	for (i = 0; i < vddc_table->count; i++) {
3764 		if (requested_voltage <= vddc_table->entries[i].v) {
3765 			requested_voltage = vddc_table->entries[i].v;
3766 			return (ci_send_msg_to_smc_with_parameter(rdev,
3767 								  PPSMC_MSG_VddC_Request,
3768 								  requested_voltage * VOLTAGE_SCALE) == PPSMC_Result_OK) ?
3769 				0 : -EINVAL;
3770 		}
3771 	}
3772 
3773 	return -EINVAL;
3774 }
3775 
ci_upload_dpm_level_enable_mask(struct radeon_device * rdev)3776 static int ci_upload_dpm_level_enable_mask(struct radeon_device *rdev)
3777 {
3778 	struct ci_power_info *pi = ci_get_pi(rdev);
3779 	PPSMC_Result result;
3780 
3781 	ci_apply_disp_minimum_voltage_request(rdev);
3782 
3783 	if (!pi->sclk_dpm_key_disabled) {
3784 		if (pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
3785 			result = ci_send_msg_to_smc_with_parameter(rdev,
3786 								   PPSMC_MSG_SCLKDPM_SetEnabledMask,
3787 								   pi->dpm_level_enable_mask.sclk_dpm_enable_mask);
3788 			if (result != PPSMC_Result_OK)
3789 				return -EINVAL;
3790 		}
3791 	}
3792 
3793 	if (!pi->mclk_dpm_key_disabled) {
3794 		if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask) {
3795 			result = ci_send_msg_to_smc_with_parameter(rdev,
3796 								   PPSMC_MSG_MCLKDPM_SetEnabledMask,
3797 								   pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
3798 			if (result != PPSMC_Result_OK)
3799 				return -EINVAL;
3800 		}
3801 	}
3802 #if 0
3803 	if (!pi->pcie_dpm_key_disabled) {
3804 		if (pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
3805 			result = ci_send_msg_to_smc_with_parameter(rdev,
3806 								   PPSMC_MSG_PCIeDPM_SetEnabledMask,
3807 								   pi->dpm_level_enable_mask.pcie_dpm_enable_mask);
3808 			if (result != PPSMC_Result_OK)
3809 				return -EINVAL;
3810 		}
3811 	}
3812 #endif
3813 	return 0;
3814 }
3815 
ci_find_dpm_states_clocks_in_dpm_table(struct radeon_device * rdev,struct radeon_ps * radeon_state)3816 static void ci_find_dpm_states_clocks_in_dpm_table(struct radeon_device *rdev,
3817 						   struct radeon_ps *radeon_state)
3818 {
3819 	struct ci_power_info *pi = ci_get_pi(rdev);
3820 	struct ci_ps *state = ci_get_ps(radeon_state);
3821 	struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table;
3822 	u32 sclk = state->performance_levels[state->performance_level_count-1].sclk;
3823 	struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table;
3824 	u32 mclk = state->performance_levels[state->performance_level_count-1].mclk;
3825 	u32 i;
3826 
3827 	pi->need_update_smu7_dpm_table = 0;
3828 
3829 	for (i = 0; i < sclk_table->count; i++) {
3830 		if (sclk == sclk_table->dpm_levels[i].value)
3831 			break;
3832 	}
3833 
3834 	if (i >= sclk_table->count) {
3835 		pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
3836 	} else {
3837 		/* XXX The current code always reprogrammed the sclk levels,
3838 		 * but we don't currently handle disp sclk requirements
3839 		 * so just skip it.
3840 		 */
3841 		if (CISLAND_MINIMUM_ENGINE_CLOCK != CISLAND_MINIMUM_ENGINE_CLOCK)
3842 			pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
3843 	}
3844 
3845 	for (i = 0; i < mclk_table->count; i++) {
3846 		if (mclk == mclk_table->dpm_levels[i].value)
3847 			break;
3848 	}
3849 
3850 	if (i >= mclk_table->count)
3851 		pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
3852 
3853 	if (rdev->pm.dpm.current_active_crtc_count !=
3854 	    rdev->pm.dpm.new_active_crtc_count)
3855 		pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
3856 }
3857 
ci_populate_and_upload_sclk_mclk_dpm_levels(struct radeon_device * rdev,struct radeon_ps * radeon_state)3858 static int ci_populate_and_upload_sclk_mclk_dpm_levels(struct radeon_device *rdev,
3859 						       struct radeon_ps *radeon_state)
3860 {
3861 	struct ci_power_info *pi = ci_get_pi(rdev);
3862 	struct ci_ps *state = ci_get_ps(radeon_state);
3863 	u32 sclk = state->performance_levels[state->performance_level_count-1].sclk;
3864 	u32 mclk = state->performance_levels[state->performance_level_count-1].mclk;
3865 	struct ci_dpm_table *dpm_table = &pi->dpm_table;
3866 	int ret;
3867 
3868 	if (!pi->need_update_smu7_dpm_table)
3869 		return 0;
3870 
3871 	if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK)
3872 		dpm_table->sclk_table.dpm_levels[dpm_table->sclk_table.count-1].value = sclk;
3873 
3874 	if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)
3875 		dpm_table->mclk_table.dpm_levels[dpm_table->mclk_table.count-1].value = mclk;
3876 
3877 	if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK)) {
3878 		ret = ci_populate_all_graphic_levels(rdev);
3879 		if (ret)
3880 			return ret;
3881 	}
3882 
3883 	if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_MCLK | DPMTABLE_UPDATE_MCLK)) {
3884 		ret = ci_populate_all_memory_levels(rdev);
3885 		if (ret)
3886 			return ret;
3887 	}
3888 
3889 	return 0;
3890 }
3891 
ci_enable_uvd_dpm(struct radeon_device * rdev,bool enable)3892 static int ci_enable_uvd_dpm(struct radeon_device *rdev, bool enable)
3893 {
3894 	struct ci_power_info *pi = ci_get_pi(rdev);
3895 	const struct radeon_clock_and_voltage_limits *max_limits;
3896 	int i;
3897 
3898 	if (rdev->pm.dpm.ac_power)
3899 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
3900 	else
3901 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
3902 
3903 	if (enable) {
3904 		pi->dpm_level_enable_mask.uvd_dpm_enable_mask = 0;
3905 
3906 		for (i = rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
3907 			if (rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
3908 				pi->dpm_level_enable_mask.uvd_dpm_enable_mask |= 1 << i;
3909 
3910 				if (!pi->caps_uvd_dpm)
3911 					break;
3912 			}
3913 		}
3914 
3915 		ci_send_msg_to_smc_with_parameter(rdev,
3916 						  PPSMC_MSG_UVDDPM_SetEnabledMask,
3917 						  pi->dpm_level_enable_mask.uvd_dpm_enable_mask);
3918 
3919 		if (pi->last_mclk_dpm_enable_mask & 0x1) {
3920 			pi->uvd_enabled = true;
3921 			pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
3922 			ci_send_msg_to_smc_with_parameter(rdev,
3923 							  PPSMC_MSG_MCLKDPM_SetEnabledMask,
3924 							  pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
3925 		}
3926 	} else {
3927 		if (pi->last_mclk_dpm_enable_mask & 0x1) {
3928 			pi->uvd_enabled = false;
3929 			pi->dpm_level_enable_mask.mclk_dpm_enable_mask |= 1;
3930 			ci_send_msg_to_smc_with_parameter(rdev,
3931 							  PPSMC_MSG_MCLKDPM_SetEnabledMask,
3932 							  pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
3933 		}
3934 	}
3935 
3936 	return (ci_send_msg_to_smc(rdev, enable ?
3937 				   PPSMC_MSG_UVDDPM_Enable : PPSMC_MSG_UVDDPM_Disable) == PPSMC_Result_OK) ?
3938 		0 : -EINVAL;
3939 }
3940 
ci_enable_vce_dpm(struct radeon_device * rdev,bool enable)3941 static int ci_enable_vce_dpm(struct radeon_device *rdev, bool enable)
3942 {
3943 	struct ci_power_info *pi = ci_get_pi(rdev);
3944 	const struct radeon_clock_and_voltage_limits *max_limits;
3945 	int i;
3946 
3947 	if (rdev->pm.dpm.ac_power)
3948 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
3949 	else
3950 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
3951 
3952 	if (enable) {
3953 		pi->dpm_level_enable_mask.vce_dpm_enable_mask = 0;
3954 		for (i = rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
3955 			if (rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
3956 				pi->dpm_level_enable_mask.vce_dpm_enable_mask |= 1 << i;
3957 
3958 				if (!pi->caps_vce_dpm)
3959 					break;
3960 			}
3961 		}
3962 
3963 		ci_send_msg_to_smc_with_parameter(rdev,
3964 						  PPSMC_MSG_VCEDPM_SetEnabledMask,
3965 						  pi->dpm_level_enable_mask.vce_dpm_enable_mask);
3966 	}
3967 
3968 	return (ci_send_msg_to_smc(rdev, enable ?
3969 				   PPSMC_MSG_VCEDPM_Enable : PPSMC_MSG_VCEDPM_Disable) == PPSMC_Result_OK) ?
3970 		0 : -EINVAL;
3971 }
3972 
3973 #if 0
3974 static int ci_enable_samu_dpm(struct radeon_device *rdev, bool enable)
3975 {
3976 	struct ci_power_info *pi = ci_get_pi(rdev);
3977 	const struct radeon_clock_and_voltage_limits *max_limits;
3978 	int i;
3979 
3980 	if (rdev->pm.dpm.ac_power)
3981 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
3982 	else
3983 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
3984 
3985 	if (enable) {
3986 		pi->dpm_level_enable_mask.samu_dpm_enable_mask = 0;
3987 		for (i = rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
3988 			if (rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
3989 				pi->dpm_level_enable_mask.samu_dpm_enable_mask |= 1 << i;
3990 
3991 				if (!pi->caps_samu_dpm)
3992 					break;
3993 			}
3994 		}
3995 
3996 		ci_send_msg_to_smc_with_parameter(rdev,
3997 						  PPSMC_MSG_SAMUDPM_SetEnabledMask,
3998 						  pi->dpm_level_enable_mask.samu_dpm_enable_mask);
3999 	}
4000 	return (ci_send_msg_to_smc(rdev, enable ?
4001 				   PPSMC_MSG_SAMUDPM_Enable : PPSMC_MSG_SAMUDPM_Disable) == PPSMC_Result_OK) ?
4002 		0 : -EINVAL;
4003 }
4004 
4005 static int ci_enable_acp_dpm(struct radeon_device *rdev, bool enable)
4006 {
4007 	struct ci_power_info *pi = ci_get_pi(rdev);
4008 	const struct radeon_clock_and_voltage_limits *max_limits;
4009 	int i;
4010 
4011 	if (rdev->pm.dpm.ac_power)
4012 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4013 	else
4014 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
4015 
4016 	if (enable) {
4017 		pi->dpm_level_enable_mask.acp_dpm_enable_mask = 0;
4018 		for (i = rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
4019 			if (rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
4020 				pi->dpm_level_enable_mask.acp_dpm_enable_mask |= 1 << i;
4021 
4022 				if (!pi->caps_acp_dpm)
4023 					break;
4024 			}
4025 		}
4026 
4027 		ci_send_msg_to_smc_with_parameter(rdev,
4028 						  PPSMC_MSG_ACPDPM_SetEnabledMask,
4029 						  pi->dpm_level_enable_mask.acp_dpm_enable_mask);
4030 	}
4031 
4032 	return (ci_send_msg_to_smc(rdev, enable ?
4033 				   PPSMC_MSG_ACPDPM_Enable : PPSMC_MSG_ACPDPM_Disable) == PPSMC_Result_OK) ?
4034 		0 : -EINVAL;
4035 }
4036 #endif
4037 
ci_update_uvd_dpm(struct radeon_device * rdev,bool gate)4038 static int ci_update_uvd_dpm(struct radeon_device *rdev, bool gate)
4039 {
4040 	struct ci_power_info *pi = ci_get_pi(rdev);
4041 	u32 tmp;
4042 
4043 	if (!gate) {
4044 		if (pi->caps_uvd_dpm ||
4045 		    (rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count <= 0))
4046 			pi->smc_state_table.UvdBootLevel = 0;
4047 		else
4048 			pi->smc_state_table.UvdBootLevel =
4049 				rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1;
4050 
4051 		tmp = RREG32_SMC(DPM_TABLE_475);
4052 		tmp &= ~UvdBootLevel_MASK;
4053 		tmp |= UvdBootLevel(pi->smc_state_table.UvdBootLevel);
4054 		WREG32_SMC(DPM_TABLE_475, tmp);
4055 	}
4056 
4057 	return ci_enable_uvd_dpm(rdev, !gate);
4058 }
4059 
ci_get_vce_boot_level(struct radeon_device * rdev)4060 static u8 ci_get_vce_boot_level(struct radeon_device *rdev)
4061 {
4062 	u8 i;
4063 	u32 min_evclk = 30000; /* ??? */
4064 	struct radeon_vce_clock_voltage_dependency_table *table =
4065 		&rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
4066 
4067 	for (i = 0; i < table->count; i++) {
4068 		if (table->entries[i].evclk >= min_evclk)
4069 			return i;
4070 	}
4071 
4072 	return table->count - 1;
4073 }
4074 
ci_update_vce_dpm(struct radeon_device * rdev,struct radeon_ps * radeon_new_state,struct radeon_ps * radeon_current_state)4075 static int ci_update_vce_dpm(struct radeon_device *rdev,
4076 			     struct radeon_ps *radeon_new_state,
4077 			     struct radeon_ps *radeon_current_state)
4078 {
4079 	struct ci_power_info *pi = ci_get_pi(rdev);
4080 	int ret = 0;
4081 	u32 tmp;
4082 
4083 	if (radeon_current_state->evclk != radeon_new_state->evclk) {
4084 		if (radeon_new_state->evclk) {
4085 			/* turn the clocks on when encoding */
4086 			cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, false);
4087 
4088 			pi->smc_state_table.VceBootLevel = ci_get_vce_boot_level(rdev);
4089 			tmp = RREG32_SMC(DPM_TABLE_475);
4090 			tmp &= ~VceBootLevel_MASK;
4091 			tmp |= VceBootLevel(pi->smc_state_table.VceBootLevel);
4092 			WREG32_SMC(DPM_TABLE_475, tmp);
4093 
4094 			ret = ci_enable_vce_dpm(rdev, true);
4095 		} else {
4096 			/* turn the clocks off when not encoding */
4097 			cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, true);
4098 
4099 			ret = ci_enable_vce_dpm(rdev, false);
4100 		}
4101 	}
4102 	return ret;
4103 }
4104 
4105 #if 0
4106 static int ci_update_samu_dpm(struct radeon_device *rdev, bool gate)
4107 {
4108 	return ci_enable_samu_dpm(rdev, gate);
4109 }
4110 
4111 static int ci_update_acp_dpm(struct radeon_device *rdev, bool gate)
4112 {
4113 	struct ci_power_info *pi = ci_get_pi(rdev);
4114 	u32 tmp;
4115 
4116 	if (!gate) {
4117 		pi->smc_state_table.AcpBootLevel = 0;
4118 
4119 		tmp = RREG32_SMC(DPM_TABLE_475);
4120 		tmp &= ~AcpBootLevel_MASK;
4121 		tmp |= AcpBootLevel(pi->smc_state_table.AcpBootLevel);
4122 		WREG32_SMC(DPM_TABLE_475, tmp);
4123 	}
4124 
4125 	return ci_enable_acp_dpm(rdev, !gate);
4126 }
4127 #endif
4128 
ci_generate_dpm_level_enable_mask(struct radeon_device * rdev,struct radeon_ps * radeon_state)4129 static int ci_generate_dpm_level_enable_mask(struct radeon_device *rdev,
4130 					     struct radeon_ps *radeon_state)
4131 {
4132 	struct ci_power_info *pi = ci_get_pi(rdev);
4133 	int ret;
4134 
4135 	ret = ci_trim_dpm_states(rdev, radeon_state);
4136 	if (ret)
4137 		return ret;
4138 
4139 	pi->dpm_level_enable_mask.sclk_dpm_enable_mask =
4140 		ci_get_dpm_level_enable_mask_value(&pi->dpm_table.sclk_table);
4141 	pi->dpm_level_enable_mask.mclk_dpm_enable_mask =
4142 		ci_get_dpm_level_enable_mask_value(&pi->dpm_table.mclk_table);
4143 	pi->last_mclk_dpm_enable_mask =
4144 		pi->dpm_level_enable_mask.mclk_dpm_enable_mask;
4145 	if (pi->uvd_enabled) {
4146 		if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask & 1)
4147 			pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
4148 	}
4149 	pi->dpm_level_enable_mask.pcie_dpm_enable_mask =
4150 		ci_get_dpm_level_enable_mask_value(&pi->dpm_table.pcie_speed_table);
4151 
4152 	return 0;
4153 }
4154 
ci_get_lowest_enabled_level(struct radeon_device * rdev,u32 level_mask)4155 static u32 ci_get_lowest_enabled_level(struct radeon_device *rdev,
4156 				       u32 level_mask)
4157 {
4158 	u32 level = 0;
4159 
4160 	while ((level_mask & (1 << level)) == 0)
4161 		level++;
4162 
4163 	return level;
4164 }
4165 
4166 
ci_dpm_force_performance_level(struct radeon_device * rdev,enum radeon_dpm_forced_level level)4167 int ci_dpm_force_performance_level(struct radeon_device *rdev,
4168 				   enum radeon_dpm_forced_level level)
4169 {
4170 	struct ci_power_info *pi = ci_get_pi(rdev);
4171 	u32 tmp, levels, i;
4172 	int ret;
4173 
4174 	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
4175 		if ((!pi->pcie_dpm_key_disabled) &&
4176 		    pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
4177 			levels = 0;
4178 			tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask;
4179 			while (tmp >>= 1)
4180 				levels++;
4181 			if (levels) {
4182 				ret = ci_dpm_force_state_pcie(rdev, level);
4183 				if (ret)
4184 					return ret;
4185 				for (i = 0; i < rdev->usec_timeout; i++) {
4186 					tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX_1) &
4187 					       CURR_PCIE_INDEX_MASK) >> CURR_PCIE_INDEX_SHIFT;
4188 					if (tmp == levels)
4189 						break;
4190 					udelay(1);
4191 				}
4192 			}
4193 		}
4194 		if ((!pi->sclk_dpm_key_disabled) &&
4195 		    pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
4196 			levels = 0;
4197 			tmp = pi->dpm_level_enable_mask.sclk_dpm_enable_mask;
4198 			while (tmp >>= 1)
4199 				levels++;
4200 			if (levels) {
4201 				ret = ci_dpm_force_state_sclk(rdev, levels);
4202 				if (ret)
4203 					return ret;
4204 				for (i = 0; i < rdev->usec_timeout; i++) {
4205 					tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) &
4206 					       CURR_SCLK_INDEX_MASK) >> CURR_SCLK_INDEX_SHIFT;
4207 					if (tmp == levels)
4208 						break;
4209 					udelay(1);
4210 				}
4211 			}
4212 		}
4213 		if ((!pi->mclk_dpm_key_disabled) &&
4214 		    pi->dpm_level_enable_mask.mclk_dpm_enable_mask) {
4215 			levels = 0;
4216 			tmp = pi->dpm_level_enable_mask.mclk_dpm_enable_mask;
4217 			while (tmp >>= 1)
4218 				levels++;
4219 			if (levels) {
4220 				ret = ci_dpm_force_state_mclk(rdev, levels);
4221 				if (ret)
4222 					return ret;
4223 				for (i = 0; i < rdev->usec_timeout; i++) {
4224 					tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) &
4225 					       CURR_MCLK_INDEX_MASK) >> CURR_MCLK_INDEX_SHIFT;
4226 					if (tmp == levels)
4227 						break;
4228 					udelay(1);
4229 				}
4230 			}
4231 		}
4232 	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
4233 		if ((!pi->sclk_dpm_key_disabled) &&
4234 		    pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
4235 			levels = ci_get_lowest_enabled_level(rdev,
4236 							     pi->dpm_level_enable_mask.sclk_dpm_enable_mask);
4237 			ret = ci_dpm_force_state_sclk(rdev, levels);
4238 			if (ret)
4239 				return ret;
4240 			for (i = 0; i < rdev->usec_timeout; i++) {
4241 				tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) &
4242 				       CURR_SCLK_INDEX_MASK) >> CURR_SCLK_INDEX_SHIFT;
4243 				if (tmp == levels)
4244 					break;
4245 				udelay(1);
4246 			}
4247 		}
4248 		if ((!pi->mclk_dpm_key_disabled) &&
4249 		    pi->dpm_level_enable_mask.mclk_dpm_enable_mask) {
4250 			levels = ci_get_lowest_enabled_level(rdev,
4251 							     pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
4252 			ret = ci_dpm_force_state_mclk(rdev, levels);
4253 			if (ret)
4254 				return ret;
4255 			for (i = 0; i < rdev->usec_timeout; i++) {
4256 				tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) &
4257 				       CURR_MCLK_INDEX_MASK) >> CURR_MCLK_INDEX_SHIFT;
4258 				if (tmp == levels)
4259 					break;
4260 				udelay(1);
4261 			}
4262 		}
4263 		if ((!pi->pcie_dpm_key_disabled) &&
4264 		    pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
4265 			levels = ci_get_lowest_enabled_level(rdev,
4266 							     pi->dpm_level_enable_mask.pcie_dpm_enable_mask);
4267 			ret = ci_dpm_force_state_pcie(rdev, levels);
4268 			if (ret)
4269 				return ret;
4270 			for (i = 0; i < rdev->usec_timeout; i++) {
4271 				tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX_1) &
4272 				       CURR_PCIE_INDEX_MASK) >> CURR_PCIE_INDEX_SHIFT;
4273 				if (tmp == levels)
4274 					break;
4275 				udelay(1);
4276 			}
4277 		}
4278 	} else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
4279 		if (!pi->pcie_dpm_key_disabled) {
4280 			PPSMC_Result smc_result;
4281 
4282 			smc_result = ci_send_msg_to_smc(rdev,
4283 							PPSMC_MSG_PCIeDPM_UnForceLevel);
4284 			if (smc_result != PPSMC_Result_OK)
4285 				return -EINVAL;
4286 		}
4287 		ret = ci_upload_dpm_level_enable_mask(rdev);
4288 		if (ret)
4289 			return ret;
4290 	}
4291 
4292 	rdev->pm.dpm.forced_level = level;
4293 
4294 	return 0;
4295 }
4296 
ci_set_mc_special_registers(struct radeon_device * rdev,struct ci_mc_reg_table * table)4297 static int ci_set_mc_special_registers(struct radeon_device *rdev,
4298 				       struct ci_mc_reg_table *table)
4299 {
4300 	struct ci_power_info *pi = ci_get_pi(rdev);
4301 	u8 i, j, k;
4302 	u32 temp_reg;
4303 
4304 	for (i = 0, j = table->last; i < table->last; i++) {
4305 		if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4306 			return -EINVAL;
4307 		switch(table->mc_reg_address[i].s1 << 2) {
4308 		case MC_SEQ_MISC1:
4309 			temp_reg = RREG32(MC_PMG_CMD_EMRS);
4310 			table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
4311 			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
4312 			for (k = 0; k < table->num_entries; k++) {
4313 				table->mc_reg_table_entry[k].mc_data[j] =
4314 					((temp_reg & 0xffff0000)) | ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
4315 			}
4316 			j++;
4317 			if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4318 				return -EINVAL;
4319 
4320 			temp_reg = RREG32(MC_PMG_CMD_MRS);
4321 			table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
4322 			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
4323 			for (k = 0; k < table->num_entries; k++) {
4324 				table->mc_reg_table_entry[k].mc_data[j] =
4325 					(temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
4326 				if (!pi->mem_gddr5)
4327 					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
4328 			}
4329 			j++;
4330 			if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4331 				return -EINVAL;
4332 
4333 			if (!pi->mem_gddr5) {
4334 				table->mc_reg_address[j].s1 = MC_PMG_AUTO_CMD >> 2;
4335 				table->mc_reg_address[j].s0 = MC_PMG_AUTO_CMD >> 2;
4336 				for (k = 0; k < table->num_entries; k++) {
4337 					table->mc_reg_table_entry[k].mc_data[j] =
4338 						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
4339 				}
4340 				j++;
4341 				if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4342 					return -EINVAL;
4343 			}
4344 			break;
4345 		case MC_SEQ_RESERVE_M:
4346 			temp_reg = RREG32(MC_PMG_CMD_MRS1);
4347 			table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
4348 			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
4349 			for (k = 0; k < table->num_entries; k++) {
4350 				table->mc_reg_table_entry[k].mc_data[j] =
4351 					(temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
4352 			}
4353 			j++;
4354 			if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4355 				return -EINVAL;
4356 			break;
4357 		default:
4358 			break;
4359 		}
4360 
4361 	}
4362 
4363 	table->last = j;
4364 
4365 	return 0;
4366 }
4367 
ci_check_s0_mc_reg_index(u16 in_reg,u16 * out_reg)4368 static bool ci_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
4369 {
4370 	bool result = true;
4371 
4372 	switch(in_reg) {
4373 	case MC_SEQ_RAS_TIMING >> 2:
4374 		*out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
4375 		break;
4376 	case MC_SEQ_DLL_STBY >> 2:
4377 		*out_reg = MC_SEQ_DLL_STBY_LP >> 2;
4378 		break;
4379 	case MC_SEQ_G5PDX_CMD0 >> 2:
4380 		*out_reg = MC_SEQ_G5PDX_CMD0_LP >> 2;
4381 		break;
4382 	case MC_SEQ_G5PDX_CMD1 >> 2:
4383 		*out_reg = MC_SEQ_G5PDX_CMD1_LP >> 2;
4384 		break;
4385 	case MC_SEQ_G5PDX_CTRL >> 2:
4386 		*out_reg = MC_SEQ_G5PDX_CTRL_LP >> 2;
4387 		break;
4388 	case MC_SEQ_CAS_TIMING >> 2:
4389 		*out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
4390 		break;
4391 	case MC_SEQ_MISC_TIMING >> 2:
4392 		*out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
4393 		break;
4394 	case MC_SEQ_MISC_TIMING2 >> 2:
4395 		*out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
4396 		break;
4397 	case MC_SEQ_PMG_DVS_CMD >> 2:
4398 		*out_reg = MC_SEQ_PMG_DVS_CMD_LP >> 2;
4399 		break;
4400 	case MC_SEQ_PMG_DVS_CTL >> 2:
4401 		*out_reg = MC_SEQ_PMG_DVS_CTL_LP >> 2;
4402 		break;
4403 	case MC_SEQ_RD_CTL_D0 >> 2:
4404 		*out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
4405 		break;
4406 	case MC_SEQ_RD_CTL_D1 >> 2:
4407 		*out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
4408 		break;
4409 	case MC_SEQ_WR_CTL_D0 >> 2:
4410 		*out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
4411 		break;
4412 	case MC_SEQ_WR_CTL_D1 >> 2:
4413 		*out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
4414 		break;
4415 	case MC_PMG_CMD_EMRS >> 2:
4416 		*out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
4417 		break;
4418 	case MC_PMG_CMD_MRS >> 2:
4419 		*out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
4420 		break;
4421 	case MC_PMG_CMD_MRS1 >> 2:
4422 		*out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
4423 		break;
4424 	case MC_SEQ_PMG_TIMING >> 2:
4425 		*out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
4426 		break;
4427 	case MC_PMG_CMD_MRS2 >> 2:
4428 		*out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
4429 		break;
4430 	case MC_SEQ_WR_CTL_2 >> 2:
4431 		*out_reg = MC_SEQ_WR_CTL_2_LP >> 2;
4432 		break;
4433 	default:
4434 		result = false;
4435 		break;
4436 	}
4437 
4438 	return result;
4439 }
4440 
ci_set_valid_flag(struct ci_mc_reg_table * table)4441 static void ci_set_valid_flag(struct ci_mc_reg_table *table)
4442 {
4443 	u8 i, j;
4444 
4445 	for (i = 0; i < table->last; i++) {
4446 		for (j = 1; j < table->num_entries; j++) {
4447 			if (table->mc_reg_table_entry[j-1].mc_data[i] !=
4448 			    table->mc_reg_table_entry[j].mc_data[i]) {
4449 				table->valid_flag |= 1 << i;
4450 				break;
4451 			}
4452 		}
4453 	}
4454 }
4455 
ci_set_s0_mc_reg_index(struct ci_mc_reg_table * table)4456 static void ci_set_s0_mc_reg_index(struct ci_mc_reg_table *table)
4457 {
4458 	u32 i;
4459 	u16 address;
4460 
4461 	for (i = 0; i < table->last; i++) {
4462 		table->mc_reg_address[i].s0 =
4463 			ci_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
4464 			address : table->mc_reg_address[i].s1;
4465 	}
4466 }
4467 
ci_copy_vbios_mc_reg_table(const struct atom_mc_reg_table * table,struct ci_mc_reg_table * ci_table)4468 static int ci_copy_vbios_mc_reg_table(const struct atom_mc_reg_table *table,
4469 				      struct ci_mc_reg_table *ci_table)
4470 {
4471 	u8 i, j;
4472 
4473 	if (table->last > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4474 		return -EINVAL;
4475 	if (table->num_entries > MAX_AC_TIMING_ENTRIES)
4476 		return -EINVAL;
4477 
4478 	for (i = 0; i < table->last; i++)
4479 		ci_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
4480 
4481 	ci_table->last = table->last;
4482 
4483 	for (i = 0; i < table->num_entries; i++) {
4484 		ci_table->mc_reg_table_entry[i].mclk_max =
4485 			table->mc_reg_table_entry[i].mclk_max;
4486 		for (j = 0; j < table->last; j++)
4487 			ci_table->mc_reg_table_entry[i].mc_data[j] =
4488 				table->mc_reg_table_entry[i].mc_data[j];
4489 	}
4490 	ci_table->num_entries = table->num_entries;
4491 
4492 	return 0;
4493 }
4494 
ci_register_patching_mc_seq(struct radeon_device * rdev,struct ci_mc_reg_table * table)4495 static int ci_register_patching_mc_seq(struct radeon_device *rdev,
4496 				       struct ci_mc_reg_table *table)
4497 {
4498 	u8 i, k;
4499 	u32 tmp;
4500 	bool patch;
4501 
4502 	tmp = RREG32(MC_SEQ_MISC0);
4503 	patch = ((tmp & 0x0000f00) == 0x300) ? true : false;
4504 
4505 	if (patch &&
4506 	    ((rdev->pdev->device == 0x67B0) ||
4507 	     (rdev->pdev->device == 0x67B1))) {
4508 		for (i = 0; i < table->last; i++) {
4509 			if (table->last >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4510 				return -EINVAL;
4511 			switch(table->mc_reg_address[i].s1 >> 2) {
4512 			case MC_SEQ_MISC1:
4513 				for (k = 0; k < table->num_entries; k++) {
4514 					if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4515 					    (table->mc_reg_table_entry[k].mclk_max == 137500))
4516 						table->mc_reg_table_entry[k].mc_data[i] =
4517 							(table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFF8) |
4518 							0x00000007;
4519 				}
4520 				break;
4521 			case MC_SEQ_WR_CTL_D0:
4522 				for (k = 0; k < table->num_entries; k++) {
4523 					if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4524 					    (table->mc_reg_table_entry[k].mclk_max == 137500))
4525 						table->mc_reg_table_entry[k].mc_data[i] =
4526 							(table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) |
4527 							0x0000D0DD;
4528 				}
4529 				break;
4530 			case MC_SEQ_WR_CTL_D1:
4531 				for (k = 0; k < table->num_entries; k++) {
4532 					if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4533 					    (table->mc_reg_table_entry[k].mclk_max == 137500))
4534 						table->mc_reg_table_entry[k].mc_data[i] =
4535 							(table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) |
4536 							0x0000D0DD;
4537 				}
4538 				break;
4539 			case MC_SEQ_WR_CTL_2:
4540 				for (k = 0; k < table->num_entries; k++) {
4541 					if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4542 					    (table->mc_reg_table_entry[k].mclk_max == 137500))
4543 						table->mc_reg_table_entry[k].mc_data[i] = 0;
4544 				}
4545 				break;
4546 			case MC_SEQ_CAS_TIMING:
4547 				for (k = 0; k < table->num_entries; k++) {
4548 					if (table->mc_reg_table_entry[k].mclk_max == 125000)
4549 						table->mc_reg_table_entry[k].mc_data[i] =
4550 							(table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) |
4551 							0x000C0140;
4552 					else if (table->mc_reg_table_entry[k].mclk_max == 137500)
4553 						table->mc_reg_table_entry[k].mc_data[i] =
4554 							(table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) |
4555 							0x000C0150;
4556 				}
4557 				break;
4558 			case MC_SEQ_MISC_TIMING:
4559 				for (k = 0; k < table->num_entries; k++) {
4560 					if (table->mc_reg_table_entry[k].mclk_max == 125000)
4561 						table->mc_reg_table_entry[k].mc_data[i] =
4562 							(table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) |
4563 							0x00000030;
4564 					else if (table->mc_reg_table_entry[k].mclk_max == 137500)
4565 						table->mc_reg_table_entry[k].mc_data[i] =
4566 							(table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) |
4567 							0x00000035;
4568 				}
4569 				break;
4570 			default:
4571 				break;
4572 			}
4573 		}
4574 
4575 		WREG32(MC_SEQ_IO_DEBUG_INDEX, 3);
4576 		tmp = RREG32(MC_SEQ_IO_DEBUG_DATA);
4577 		tmp = (tmp & 0xFFF8FFFF) | (1 << 16);
4578 		WREG32(MC_SEQ_IO_DEBUG_INDEX, 3);
4579 		WREG32(MC_SEQ_IO_DEBUG_DATA, tmp);
4580 	}
4581 
4582 	return 0;
4583 }
4584 
ci_initialize_mc_reg_table(struct radeon_device * rdev)4585 static int ci_initialize_mc_reg_table(struct radeon_device *rdev)
4586 {
4587 	struct ci_power_info *pi = ci_get_pi(rdev);
4588 	struct atom_mc_reg_table *table;
4589 	struct ci_mc_reg_table *ci_table = &pi->mc_reg_table;
4590 	u8 module_index = rv770_get_memory_module_index(rdev);
4591 	int ret;
4592 
4593 	table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
4594 	if (!table)
4595 		return -ENOMEM;
4596 
4597 	WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
4598 	WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
4599 	WREG32(MC_SEQ_DLL_STBY_LP, RREG32(MC_SEQ_DLL_STBY));
4600 	WREG32(MC_SEQ_G5PDX_CMD0_LP, RREG32(MC_SEQ_G5PDX_CMD0));
4601 	WREG32(MC_SEQ_G5PDX_CMD1_LP, RREG32(MC_SEQ_G5PDX_CMD1));
4602 	WREG32(MC_SEQ_G5PDX_CTRL_LP, RREG32(MC_SEQ_G5PDX_CTRL));
4603 	WREG32(MC_SEQ_PMG_DVS_CMD_LP, RREG32(MC_SEQ_PMG_DVS_CMD));
4604 	WREG32(MC_SEQ_PMG_DVS_CTL_LP, RREG32(MC_SEQ_PMG_DVS_CTL));
4605 	WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
4606 	WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
4607 	WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
4608 	WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
4609 	WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
4610 	WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
4611 	WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
4612 	WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
4613 	WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
4614 	WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
4615 	WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
4616 	WREG32(MC_SEQ_WR_CTL_2_LP, RREG32(MC_SEQ_WR_CTL_2));
4617 
4618 	ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
4619 	if (ret)
4620 		goto init_mc_done;
4621 
4622 	ret = ci_copy_vbios_mc_reg_table(table, ci_table);
4623 	if (ret)
4624 		goto init_mc_done;
4625 
4626 	ci_set_s0_mc_reg_index(ci_table);
4627 
4628 	ret = ci_register_patching_mc_seq(rdev, ci_table);
4629 	if (ret)
4630 		goto init_mc_done;
4631 
4632 	ret = ci_set_mc_special_registers(rdev, ci_table);
4633 	if (ret)
4634 		goto init_mc_done;
4635 
4636 	ci_set_valid_flag(ci_table);
4637 
4638 init_mc_done:
4639 	kfree(table);
4640 
4641 	return ret;
4642 }
4643 
ci_populate_mc_reg_addresses(struct radeon_device * rdev,SMU7_Discrete_MCRegisters * mc_reg_table)4644 static int ci_populate_mc_reg_addresses(struct radeon_device *rdev,
4645 					SMU7_Discrete_MCRegisters *mc_reg_table)
4646 {
4647 	struct ci_power_info *pi = ci_get_pi(rdev);
4648 	u32 i, j;
4649 
4650 	for (i = 0, j = 0; j < pi->mc_reg_table.last; j++) {
4651 		if (pi->mc_reg_table.valid_flag & (1 << j)) {
4652 			if (i >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4653 				return -EINVAL;
4654 			mc_reg_table->address[i].s0 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s0);
4655 			mc_reg_table->address[i].s1 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s1);
4656 			i++;
4657 		}
4658 	}
4659 
4660 	mc_reg_table->last = (u8)i;
4661 
4662 	return 0;
4663 }
4664 
ci_convert_mc_registers(const struct ci_mc_reg_entry * entry,SMU7_Discrete_MCRegisterSet * data,u32 num_entries,u32 valid_flag)4665 static void ci_convert_mc_registers(const struct ci_mc_reg_entry *entry,
4666 				    SMU7_Discrete_MCRegisterSet *data,
4667 				    u32 num_entries, u32 valid_flag)
4668 {
4669 	u32 i, j;
4670 
4671 	for (i = 0, j = 0; j < num_entries; j++) {
4672 		if (valid_flag & (1 << j)) {
4673 			data->value[i] = cpu_to_be32(entry->mc_data[j]);
4674 			i++;
4675 		}
4676 	}
4677 }
4678 
ci_convert_mc_reg_table_entry_to_smc(struct radeon_device * rdev,const u32 memory_clock,SMU7_Discrete_MCRegisterSet * mc_reg_table_data)4679 static void ci_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
4680 						 const u32 memory_clock,
4681 						 SMU7_Discrete_MCRegisterSet *mc_reg_table_data)
4682 {
4683 	struct ci_power_info *pi = ci_get_pi(rdev);
4684 	u32 i = 0;
4685 
4686 	for(i = 0; i < pi->mc_reg_table.num_entries; i++) {
4687 		if (memory_clock <= pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
4688 			break;
4689 	}
4690 
4691 	if ((i == pi->mc_reg_table.num_entries) && (i > 0))
4692 		--i;
4693 
4694 	ci_convert_mc_registers(&pi->mc_reg_table.mc_reg_table_entry[i],
4695 				mc_reg_table_data, pi->mc_reg_table.last,
4696 				pi->mc_reg_table.valid_flag);
4697 }
4698 
ci_convert_mc_reg_table_to_smc(struct radeon_device * rdev,SMU7_Discrete_MCRegisters * mc_reg_table)4699 static void ci_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
4700 					   SMU7_Discrete_MCRegisters *mc_reg_table)
4701 {
4702 	struct ci_power_info *pi = ci_get_pi(rdev);
4703 	u32 i;
4704 
4705 	for (i = 0; i < pi->dpm_table.mclk_table.count; i++)
4706 		ci_convert_mc_reg_table_entry_to_smc(rdev,
4707 						     pi->dpm_table.mclk_table.dpm_levels[i].value,
4708 						     &mc_reg_table->data[i]);
4709 }
4710 
ci_populate_initial_mc_reg_table(struct radeon_device * rdev)4711 static int ci_populate_initial_mc_reg_table(struct radeon_device *rdev)
4712 {
4713 	struct ci_power_info *pi = ci_get_pi(rdev);
4714 	int ret;
4715 
4716 	memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters));
4717 
4718 	ret = ci_populate_mc_reg_addresses(rdev, &pi->smc_mc_reg_table);
4719 	if (ret)
4720 		return ret;
4721 	ci_convert_mc_reg_table_to_smc(rdev, &pi->smc_mc_reg_table);
4722 
4723 	return ci_copy_bytes_to_smc(rdev,
4724 				    pi->mc_reg_table_start,
4725 				    (u8 *)&pi->smc_mc_reg_table,
4726 				    sizeof(SMU7_Discrete_MCRegisters),
4727 				    pi->sram_end);
4728 }
4729 
ci_update_and_upload_mc_reg_table(struct radeon_device * rdev)4730 static int ci_update_and_upload_mc_reg_table(struct radeon_device *rdev)
4731 {
4732 	struct ci_power_info *pi = ci_get_pi(rdev);
4733 
4734 	if (!(pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK))
4735 		return 0;
4736 
4737 	memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters));
4738 
4739 	ci_convert_mc_reg_table_to_smc(rdev, &pi->smc_mc_reg_table);
4740 
4741 	return ci_copy_bytes_to_smc(rdev,
4742 				    pi->mc_reg_table_start +
4743 				    offsetof(SMU7_Discrete_MCRegisters, data[0]),
4744 				    (u8 *)&pi->smc_mc_reg_table.data[0],
4745 				    sizeof(SMU7_Discrete_MCRegisterSet) *
4746 				    pi->dpm_table.mclk_table.count,
4747 				    pi->sram_end);
4748 }
4749 
ci_enable_voltage_control(struct radeon_device * rdev)4750 static void ci_enable_voltage_control(struct radeon_device *rdev)
4751 {
4752 	u32 tmp = RREG32_SMC(GENERAL_PWRMGT);
4753 
4754 	tmp |= VOLT_PWRMGT_EN;
4755 	WREG32_SMC(GENERAL_PWRMGT, tmp);
4756 }
4757 
ci_get_maximum_link_speed(struct radeon_device * rdev,struct radeon_ps * radeon_state)4758 static enum radeon_pcie_gen ci_get_maximum_link_speed(struct radeon_device *rdev,
4759 						      struct radeon_ps *radeon_state)
4760 {
4761 	struct ci_ps *state = ci_get_ps(radeon_state);
4762 	int i;
4763 	u16 pcie_speed, max_speed = 0;
4764 
4765 	for (i = 0; i < state->performance_level_count; i++) {
4766 		pcie_speed = state->performance_levels[i].pcie_gen;
4767 		if (max_speed < pcie_speed)
4768 			max_speed = pcie_speed;
4769 	}
4770 
4771 	return max_speed;
4772 }
4773 
ci_get_current_pcie_speed(struct radeon_device * rdev)4774 static u16 ci_get_current_pcie_speed(struct radeon_device *rdev)
4775 {
4776 	u32 speed_cntl = 0;
4777 
4778 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE_MASK;
4779 	speed_cntl >>= LC_CURRENT_DATA_RATE_SHIFT;
4780 
4781 	return (u16)speed_cntl;
4782 }
4783 
ci_get_current_pcie_lane_number(struct radeon_device * rdev)4784 static int ci_get_current_pcie_lane_number(struct radeon_device *rdev)
4785 {
4786 	u32 link_width = 0;
4787 
4788 	link_width = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL) & LC_LINK_WIDTH_RD_MASK;
4789 	link_width >>= LC_LINK_WIDTH_RD_SHIFT;
4790 
4791 	switch (link_width) {
4792 	case RADEON_PCIE_LC_LINK_WIDTH_X1:
4793 		return 1;
4794 	case RADEON_PCIE_LC_LINK_WIDTH_X2:
4795 		return 2;
4796 	case RADEON_PCIE_LC_LINK_WIDTH_X4:
4797 		return 4;
4798 	case RADEON_PCIE_LC_LINK_WIDTH_X8:
4799 		return 8;
4800 	case RADEON_PCIE_LC_LINK_WIDTH_X12:
4801 		/* not actually supported */
4802 		return 12;
4803 	case RADEON_PCIE_LC_LINK_WIDTH_X0:
4804 	case RADEON_PCIE_LC_LINK_WIDTH_X16:
4805 	default:
4806 		return 16;
4807 	}
4808 }
4809 
ci_request_link_speed_change_before_state_change(struct radeon_device * rdev,struct radeon_ps * radeon_new_state,struct radeon_ps * radeon_current_state)4810 static void ci_request_link_speed_change_before_state_change(struct radeon_device *rdev,
4811 							     struct radeon_ps *radeon_new_state,
4812 							     struct radeon_ps *radeon_current_state)
4813 {
4814 	struct ci_power_info *pi = ci_get_pi(rdev);
4815 	enum radeon_pcie_gen target_link_speed =
4816 		ci_get_maximum_link_speed(rdev, radeon_new_state);
4817 	enum radeon_pcie_gen current_link_speed;
4818 
4819 	if (pi->force_pcie_gen == RADEON_PCIE_GEN_INVALID)
4820 		current_link_speed = ci_get_maximum_link_speed(rdev, radeon_current_state);
4821 	else
4822 		current_link_speed = pi->force_pcie_gen;
4823 
4824 	pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID;
4825 	pi->pspp_notify_required = false;
4826 	if (target_link_speed > current_link_speed) {
4827 		switch (target_link_speed) {
4828 #ifdef CONFIG_ACPI
4829 		case RADEON_PCIE_GEN3:
4830 			if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN3, false) == 0)
4831 				break;
4832 			pi->force_pcie_gen = RADEON_PCIE_GEN2;
4833 			if (current_link_speed == RADEON_PCIE_GEN2)
4834 				break;
4835 			fallthrough;
4836 		case RADEON_PCIE_GEN2:
4837 			if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, false) == 0)
4838 				break;
4839 			fallthrough;
4840 #endif
4841 		default:
4842 			pi->force_pcie_gen = ci_get_current_pcie_speed(rdev);
4843 			break;
4844 		}
4845 	} else {
4846 		if (target_link_speed < current_link_speed)
4847 			pi->pspp_notify_required = true;
4848 	}
4849 }
4850 
ci_notify_link_speed_change_after_state_change(struct radeon_device * rdev,struct radeon_ps * radeon_new_state,struct radeon_ps * radeon_current_state)4851 static void ci_notify_link_speed_change_after_state_change(struct radeon_device *rdev,
4852 							   struct radeon_ps *radeon_new_state,
4853 							   struct radeon_ps *radeon_current_state)
4854 {
4855 	struct ci_power_info *pi = ci_get_pi(rdev);
4856 	enum radeon_pcie_gen target_link_speed =
4857 		ci_get_maximum_link_speed(rdev, radeon_new_state);
4858 	u8 request;
4859 
4860 	if (pi->pspp_notify_required) {
4861 		if (target_link_speed == RADEON_PCIE_GEN3)
4862 			request = PCIE_PERF_REQ_PECI_GEN3;
4863 		else if (target_link_speed == RADEON_PCIE_GEN2)
4864 			request = PCIE_PERF_REQ_PECI_GEN2;
4865 		else
4866 			request = PCIE_PERF_REQ_PECI_GEN1;
4867 
4868 		if ((request == PCIE_PERF_REQ_PECI_GEN1) &&
4869 		    (ci_get_current_pcie_speed(rdev) > 0))
4870 			return;
4871 
4872 #ifdef CONFIG_ACPI
4873 		radeon_acpi_pcie_performance_request(rdev, request, false);
4874 #endif
4875 	}
4876 }
4877 
ci_set_private_data_variables_based_on_pptable(struct radeon_device * rdev)4878 static int ci_set_private_data_variables_based_on_pptable(struct radeon_device *rdev)
4879 {
4880 	struct ci_power_info *pi = ci_get_pi(rdev);
4881 	struct radeon_clock_voltage_dependency_table *allowed_sclk_vddc_table =
4882 		&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
4883 	struct radeon_clock_voltage_dependency_table *allowed_mclk_vddc_table =
4884 		&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk;
4885 	struct radeon_clock_voltage_dependency_table *allowed_mclk_vddci_table =
4886 		&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk;
4887 
4888 	if (allowed_sclk_vddc_table == NULL)
4889 		return -EINVAL;
4890 	if (allowed_sclk_vddc_table->count < 1)
4891 		return -EINVAL;
4892 	if (allowed_mclk_vddc_table == NULL)
4893 		return -EINVAL;
4894 	if (allowed_mclk_vddc_table->count < 1)
4895 		return -EINVAL;
4896 	if (allowed_mclk_vddci_table == NULL)
4897 		return -EINVAL;
4898 	if (allowed_mclk_vddci_table->count < 1)
4899 		return -EINVAL;
4900 
4901 	pi->min_vddc_in_pp_table = allowed_sclk_vddc_table->entries[0].v;
4902 	pi->max_vddc_in_pp_table =
4903 		allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v;
4904 
4905 	pi->min_vddci_in_pp_table = allowed_mclk_vddci_table->entries[0].v;
4906 	pi->max_vddci_in_pp_table =
4907 		allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v;
4908 
4909 	rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk =
4910 		allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk;
4911 	rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk =
4912 		allowed_mclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk;
4913 	rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc =
4914 		allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v;
4915 	rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci =
4916 		allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v;
4917 
4918 	return 0;
4919 }
4920 
ci_patch_with_vddc_leakage(struct radeon_device * rdev,u16 * vddc)4921 static void ci_patch_with_vddc_leakage(struct radeon_device *rdev, u16 *vddc)
4922 {
4923 	struct ci_power_info *pi = ci_get_pi(rdev);
4924 	struct ci_leakage_voltage *leakage_table = &pi->vddc_leakage;
4925 	u32 leakage_index;
4926 
4927 	for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) {
4928 		if (leakage_table->leakage_id[leakage_index] == *vddc) {
4929 			*vddc = leakage_table->actual_voltage[leakage_index];
4930 			break;
4931 		}
4932 	}
4933 }
4934 
ci_patch_with_vddci_leakage(struct radeon_device * rdev,u16 * vddci)4935 static void ci_patch_with_vddci_leakage(struct radeon_device *rdev, u16 *vddci)
4936 {
4937 	struct ci_power_info *pi = ci_get_pi(rdev);
4938 	struct ci_leakage_voltage *leakage_table = &pi->vddci_leakage;
4939 	u32 leakage_index;
4940 
4941 	for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) {
4942 		if (leakage_table->leakage_id[leakage_index] == *vddci) {
4943 			*vddci = leakage_table->actual_voltage[leakage_index];
4944 			break;
4945 		}
4946 	}
4947 }
4948 
ci_patch_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device * rdev,struct radeon_clock_voltage_dependency_table * table)4949 static void ci_patch_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device *rdev,
4950 								      struct radeon_clock_voltage_dependency_table *table)
4951 {
4952 	u32 i;
4953 
4954 	if (table) {
4955 		for (i = 0; i < table->count; i++)
4956 			ci_patch_with_vddc_leakage(rdev, &table->entries[i].v);
4957 	}
4958 }
4959 
ci_patch_clock_voltage_dependency_table_with_vddci_leakage(struct radeon_device * rdev,struct radeon_clock_voltage_dependency_table * table)4960 static void ci_patch_clock_voltage_dependency_table_with_vddci_leakage(struct radeon_device *rdev,
4961 								       struct radeon_clock_voltage_dependency_table *table)
4962 {
4963 	u32 i;
4964 
4965 	if (table) {
4966 		for (i = 0; i < table->count; i++)
4967 			ci_patch_with_vddci_leakage(rdev, &table->entries[i].v);
4968 	}
4969 }
4970 
ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device * rdev,struct radeon_vce_clock_voltage_dependency_table * table)4971 static void ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device *rdev,
4972 									  struct radeon_vce_clock_voltage_dependency_table *table)
4973 {
4974 	u32 i;
4975 
4976 	if (table) {
4977 		for (i = 0; i < table->count; i++)
4978 			ci_patch_with_vddc_leakage(rdev, &table->entries[i].v);
4979 	}
4980 }
4981 
ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device * rdev,struct radeon_uvd_clock_voltage_dependency_table * table)4982 static void ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device *rdev,
4983 									  struct radeon_uvd_clock_voltage_dependency_table *table)
4984 {
4985 	u32 i;
4986 
4987 	if (table) {
4988 		for (i = 0; i < table->count; i++)
4989 			ci_patch_with_vddc_leakage(rdev, &table->entries[i].v);
4990 	}
4991 }
4992 
ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(struct radeon_device * rdev,struct radeon_phase_shedding_limits_table * table)4993 static void ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(struct radeon_device *rdev,
4994 								   struct radeon_phase_shedding_limits_table *table)
4995 {
4996 	u32 i;
4997 
4998 	if (table) {
4999 		for (i = 0; i < table->count; i++)
5000 			ci_patch_with_vddc_leakage(rdev, &table->entries[i].voltage);
5001 	}
5002 }
5003 
ci_patch_clock_voltage_limits_with_vddc_leakage(struct radeon_device * rdev,struct radeon_clock_and_voltage_limits * table)5004 static void ci_patch_clock_voltage_limits_with_vddc_leakage(struct radeon_device *rdev,
5005 							    struct radeon_clock_and_voltage_limits *table)
5006 {
5007 	if (table) {
5008 		ci_patch_with_vddc_leakage(rdev, (u16 *)&table->vddc);
5009 		ci_patch_with_vddci_leakage(rdev, (u16 *)&table->vddci);
5010 	}
5011 }
5012 
ci_patch_cac_leakage_table_with_vddc_leakage(struct radeon_device * rdev,struct radeon_cac_leakage_table * table)5013 static void ci_patch_cac_leakage_table_with_vddc_leakage(struct radeon_device *rdev,
5014 							 struct radeon_cac_leakage_table *table)
5015 {
5016 	u32 i;
5017 
5018 	if (table) {
5019 		for (i = 0; i < table->count; i++)
5020 			ci_patch_with_vddc_leakage(rdev, &table->entries[i].vddc);
5021 	}
5022 }
5023 
ci_patch_dependency_tables_with_leakage(struct radeon_device * rdev)5024 static void ci_patch_dependency_tables_with_leakage(struct radeon_device *rdev)
5025 {
5026 
5027 	ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5028 								  &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
5029 	ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5030 								  &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
5031 	ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5032 								  &rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk);
5033 	ci_patch_clock_voltage_dependency_table_with_vddci_leakage(rdev,
5034 								   &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk);
5035 	ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5036 								      &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table);
5037 	ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5038 								      &rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table);
5039 	ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5040 								  &rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table);
5041 	ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5042 								  &rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table);
5043 	ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(rdev,
5044 							       &rdev->pm.dpm.dyn_state.phase_shedding_limits_table);
5045 	ci_patch_clock_voltage_limits_with_vddc_leakage(rdev,
5046 							&rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac);
5047 	ci_patch_clock_voltage_limits_with_vddc_leakage(rdev,
5048 							&rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc);
5049 	ci_patch_cac_leakage_table_with_vddc_leakage(rdev,
5050 						     &rdev->pm.dpm.dyn_state.cac_leakage_table);
5051 
5052 }
5053 
ci_get_memory_type(struct radeon_device * rdev)5054 static void ci_get_memory_type(struct radeon_device *rdev)
5055 {
5056 	struct ci_power_info *pi = ci_get_pi(rdev);
5057 	u32 tmp;
5058 
5059 	tmp = RREG32(MC_SEQ_MISC0);
5060 
5061 	if (((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT) ==
5062 	    MC_SEQ_MISC0_GDDR5_VALUE)
5063 		pi->mem_gddr5 = true;
5064 	else
5065 		pi->mem_gddr5 = false;
5066 
5067 }
5068 
ci_update_current_ps(struct radeon_device * rdev,struct radeon_ps * rps)5069 static void ci_update_current_ps(struct radeon_device *rdev,
5070 				 struct radeon_ps *rps)
5071 {
5072 	struct ci_ps *new_ps = ci_get_ps(rps);
5073 	struct ci_power_info *pi = ci_get_pi(rdev);
5074 
5075 	pi->current_rps = *rps;
5076 	pi->current_ps = *new_ps;
5077 	pi->current_rps.ps_priv = &pi->current_ps;
5078 }
5079 
ci_update_requested_ps(struct radeon_device * rdev,struct radeon_ps * rps)5080 static void ci_update_requested_ps(struct radeon_device *rdev,
5081 				   struct radeon_ps *rps)
5082 {
5083 	struct ci_ps *new_ps = ci_get_ps(rps);
5084 	struct ci_power_info *pi = ci_get_pi(rdev);
5085 
5086 	pi->requested_rps = *rps;
5087 	pi->requested_ps = *new_ps;
5088 	pi->requested_rps.ps_priv = &pi->requested_ps;
5089 }
5090 
ci_dpm_pre_set_power_state(struct radeon_device * rdev)5091 int ci_dpm_pre_set_power_state(struct radeon_device *rdev)
5092 {
5093 	struct ci_power_info *pi = ci_get_pi(rdev);
5094 	struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
5095 	struct radeon_ps *new_ps = &requested_ps;
5096 
5097 	ci_update_requested_ps(rdev, new_ps);
5098 
5099 	ci_apply_state_adjust_rules(rdev, &pi->requested_rps);
5100 
5101 	return 0;
5102 }
5103 
ci_dpm_post_set_power_state(struct radeon_device * rdev)5104 void ci_dpm_post_set_power_state(struct radeon_device *rdev)
5105 {
5106 	struct ci_power_info *pi = ci_get_pi(rdev);
5107 	struct radeon_ps *new_ps = &pi->requested_rps;
5108 
5109 	ci_update_current_ps(rdev, new_ps);
5110 }
5111 
5112 
ci_dpm_setup_asic(struct radeon_device * rdev)5113 void ci_dpm_setup_asic(struct radeon_device *rdev)
5114 {
5115 	int r;
5116 
5117 	r = ci_mc_load_microcode(rdev);
5118 	if (r)
5119 		DRM_ERROR("Failed to load MC firmware!\n");
5120 	ci_read_clock_registers(rdev);
5121 	ci_get_memory_type(rdev);
5122 	ci_enable_acpi_power_management(rdev);
5123 	ci_init_sclk_t(rdev);
5124 }
5125 
ci_dpm_enable(struct radeon_device * rdev)5126 int ci_dpm_enable(struct radeon_device *rdev)
5127 {
5128 	struct ci_power_info *pi = ci_get_pi(rdev);
5129 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
5130 	int ret;
5131 
5132 	if (ci_is_smc_running(rdev))
5133 		return -EINVAL;
5134 	if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
5135 		ci_enable_voltage_control(rdev);
5136 		ret = ci_construct_voltage_tables(rdev);
5137 		if (ret) {
5138 			DRM_ERROR("ci_construct_voltage_tables failed\n");
5139 			return ret;
5140 		}
5141 	}
5142 	if (pi->caps_dynamic_ac_timing) {
5143 		ret = ci_initialize_mc_reg_table(rdev);
5144 		if (ret)
5145 			pi->caps_dynamic_ac_timing = false;
5146 	}
5147 	if (pi->dynamic_ss)
5148 		ci_enable_spread_spectrum(rdev, true);
5149 	if (pi->thermal_protection)
5150 		ci_enable_thermal_protection(rdev, true);
5151 	ci_program_sstp(rdev);
5152 	ci_enable_display_gap(rdev);
5153 	ci_program_vc(rdev);
5154 	ret = ci_upload_firmware(rdev);
5155 	if (ret) {
5156 		DRM_ERROR("ci_upload_firmware failed\n");
5157 		return ret;
5158 	}
5159 	ret = ci_process_firmware_header(rdev);
5160 	if (ret) {
5161 		DRM_ERROR("ci_process_firmware_header failed\n");
5162 		return ret;
5163 	}
5164 	ret = ci_initial_switch_from_arb_f0_to_f1(rdev);
5165 	if (ret) {
5166 		DRM_ERROR("ci_initial_switch_from_arb_f0_to_f1 failed\n");
5167 		return ret;
5168 	}
5169 	ret = ci_init_smc_table(rdev);
5170 	if (ret) {
5171 		DRM_ERROR("ci_init_smc_table failed\n");
5172 		return ret;
5173 	}
5174 	ret = ci_init_arb_table_index(rdev);
5175 	if (ret) {
5176 		DRM_ERROR("ci_init_arb_table_index failed\n");
5177 		return ret;
5178 	}
5179 	if (pi->caps_dynamic_ac_timing) {
5180 		ret = ci_populate_initial_mc_reg_table(rdev);
5181 		if (ret) {
5182 			DRM_ERROR("ci_populate_initial_mc_reg_table failed\n");
5183 			return ret;
5184 		}
5185 	}
5186 	ret = ci_populate_pm_base(rdev);
5187 	if (ret) {
5188 		DRM_ERROR("ci_populate_pm_base failed\n");
5189 		return ret;
5190 	}
5191 	ci_dpm_start_smc(rdev);
5192 	ci_enable_vr_hot_gpio_interrupt(rdev);
5193 	ret = ci_notify_smc_display_change(rdev, false);
5194 	if (ret) {
5195 		DRM_ERROR("ci_notify_smc_display_change failed\n");
5196 		return ret;
5197 	}
5198 	ci_enable_sclk_control(rdev, true);
5199 	ret = ci_enable_ulv(rdev, true);
5200 	if (ret) {
5201 		DRM_ERROR("ci_enable_ulv failed\n");
5202 		return ret;
5203 	}
5204 	ret = ci_enable_ds_master_switch(rdev, true);
5205 	if (ret) {
5206 		DRM_ERROR("ci_enable_ds_master_switch failed\n");
5207 		return ret;
5208 	}
5209 	ret = ci_start_dpm(rdev);
5210 	if (ret) {
5211 		DRM_ERROR("ci_start_dpm failed\n");
5212 		return ret;
5213 	}
5214 	ret = ci_enable_didt(rdev, true);
5215 	if (ret) {
5216 		DRM_ERROR("ci_enable_didt failed\n");
5217 		return ret;
5218 	}
5219 	ret = ci_enable_smc_cac(rdev, true);
5220 	if (ret) {
5221 		DRM_ERROR("ci_enable_smc_cac failed\n");
5222 		return ret;
5223 	}
5224 	ret = ci_enable_power_containment(rdev, true);
5225 	if (ret) {
5226 		DRM_ERROR("ci_enable_power_containment failed\n");
5227 		return ret;
5228 	}
5229 
5230 	ret = ci_power_control_set_level(rdev);
5231 	if (ret) {
5232 		DRM_ERROR("ci_power_control_set_level failed\n");
5233 		return ret;
5234 	}
5235 
5236 	ci_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
5237 
5238 	ret = ci_enable_thermal_based_sclk_dpm(rdev, true);
5239 	if (ret) {
5240 		DRM_ERROR("ci_enable_thermal_based_sclk_dpm failed\n");
5241 		return ret;
5242 	}
5243 
5244 	ci_thermal_start_thermal_controller(rdev);
5245 
5246 	ci_update_current_ps(rdev, boot_ps);
5247 
5248 	return 0;
5249 }
5250 
ci_set_temperature_range(struct radeon_device * rdev)5251 static int ci_set_temperature_range(struct radeon_device *rdev)
5252 {
5253 	int ret;
5254 
5255 	ret = ci_thermal_enable_alert(rdev, false);
5256 	if (ret)
5257 		return ret;
5258 	ret = ci_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
5259 	if (ret)
5260 		return ret;
5261 	ret = ci_thermal_enable_alert(rdev, true);
5262 	if (ret)
5263 		return ret;
5264 
5265 	return ret;
5266 }
5267 
ci_dpm_late_enable(struct radeon_device * rdev)5268 int ci_dpm_late_enable(struct radeon_device *rdev)
5269 {
5270 	int ret;
5271 
5272 	ret = ci_set_temperature_range(rdev);
5273 	if (ret)
5274 		return ret;
5275 
5276 	ci_dpm_powergate_uvd(rdev, true);
5277 
5278 	return 0;
5279 }
5280 
ci_dpm_disable(struct radeon_device * rdev)5281 void ci_dpm_disable(struct radeon_device *rdev)
5282 {
5283 	struct ci_power_info *pi = ci_get_pi(rdev);
5284 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
5285 
5286 	ci_dpm_powergate_uvd(rdev, false);
5287 
5288 	if (!ci_is_smc_running(rdev))
5289 		return;
5290 
5291 	ci_thermal_stop_thermal_controller(rdev);
5292 
5293 	if (pi->thermal_protection)
5294 		ci_enable_thermal_protection(rdev, false);
5295 	ci_enable_power_containment(rdev, false);
5296 	ci_enable_smc_cac(rdev, false);
5297 	ci_enable_didt(rdev, false);
5298 	ci_enable_spread_spectrum(rdev, false);
5299 	ci_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
5300 	ci_stop_dpm(rdev);
5301 	ci_enable_ds_master_switch(rdev, false);
5302 	ci_enable_ulv(rdev, false);
5303 	ci_clear_vc(rdev);
5304 	ci_reset_to_default(rdev);
5305 	ci_dpm_stop_smc(rdev);
5306 	ci_force_switch_to_arb_f0(rdev);
5307 	ci_enable_thermal_based_sclk_dpm(rdev, false);
5308 
5309 	ci_update_current_ps(rdev, boot_ps);
5310 }
5311 
ci_dpm_set_power_state(struct radeon_device * rdev)5312 int ci_dpm_set_power_state(struct radeon_device *rdev)
5313 {
5314 	struct ci_power_info *pi = ci_get_pi(rdev);
5315 	struct radeon_ps *new_ps = &pi->requested_rps;
5316 	struct radeon_ps *old_ps = &pi->current_rps;
5317 	int ret;
5318 
5319 	ci_find_dpm_states_clocks_in_dpm_table(rdev, new_ps);
5320 	if (pi->pcie_performance_request)
5321 		ci_request_link_speed_change_before_state_change(rdev, new_ps, old_ps);
5322 	ret = ci_freeze_sclk_mclk_dpm(rdev);
5323 	if (ret) {
5324 		DRM_ERROR("ci_freeze_sclk_mclk_dpm failed\n");
5325 		return ret;
5326 	}
5327 	ret = ci_populate_and_upload_sclk_mclk_dpm_levels(rdev, new_ps);
5328 	if (ret) {
5329 		DRM_ERROR("ci_populate_and_upload_sclk_mclk_dpm_levels failed\n");
5330 		return ret;
5331 	}
5332 	ret = ci_generate_dpm_level_enable_mask(rdev, new_ps);
5333 	if (ret) {
5334 		DRM_ERROR("ci_generate_dpm_level_enable_mask failed\n");
5335 		return ret;
5336 	}
5337 
5338 	ret = ci_update_vce_dpm(rdev, new_ps, old_ps);
5339 	if (ret) {
5340 		DRM_ERROR("ci_update_vce_dpm failed\n");
5341 		return ret;
5342 	}
5343 
5344 	ret = ci_update_sclk_t(rdev);
5345 	if (ret) {
5346 		DRM_ERROR("ci_update_sclk_t failed\n");
5347 		return ret;
5348 	}
5349 	if (pi->caps_dynamic_ac_timing) {
5350 		ret = ci_update_and_upload_mc_reg_table(rdev);
5351 		if (ret) {
5352 			DRM_ERROR("ci_update_and_upload_mc_reg_table failed\n");
5353 			return ret;
5354 		}
5355 	}
5356 	ret = ci_program_memory_timing_parameters(rdev);
5357 	if (ret) {
5358 		DRM_ERROR("ci_program_memory_timing_parameters failed\n");
5359 		return ret;
5360 	}
5361 	ret = ci_unfreeze_sclk_mclk_dpm(rdev);
5362 	if (ret) {
5363 		DRM_ERROR("ci_unfreeze_sclk_mclk_dpm failed\n");
5364 		return ret;
5365 	}
5366 	ret = ci_upload_dpm_level_enable_mask(rdev);
5367 	if (ret) {
5368 		DRM_ERROR("ci_upload_dpm_level_enable_mask failed\n");
5369 		return ret;
5370 	}
5371 	if (pi->pcie_performance_request)
5372 		ci_notify_link_speed_change_after_state_change(rdev, new_ps, old_ps);
5373 
5374 	return 0;
5375 }
5376 
5377 #if 0
5378 void ci_dpm_reset_asic(struct radeon_device *rdev)
5379 {
5380 	ci_set_boot_state(rdev);
5381 }
5382 #endif
5383 
ci_dpm_display_configuration_changed(struct radeon_device * rdev)5384 void ci_dpm_display_configuration_changed(struct radeon_device *rdev)
5385 {
5386 	ci_program_display_gap(rdev);
5387 }
5388 
5389 union power_info {
5390 	struct _ATOM_POWERPLAY_INFO info;
5391 	struct _ATOM_POWERPLAY_INFO_V2 info_2;
5392 	struct _ATOM_POWERPLAY_INFO_V3 info_3;
5393 	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
5394 	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
5395 	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
5396 };
5397 
5398 union pplib_clock_info {
5399 	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
5400 	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
5401 	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
5402 	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
5403 	struct _ATOM_PPLIB_SI_CLOCK_INFO si;
5404 	struct _ATOM_PPLIB_CI_CLOCK_INFO ci;
5405 };
5406 
5407 union pplib_power_state {
5408 	struct _ATOM_PPLIB_STATE v1;
5409 	struct _ATOM_PPLIB_STATE_V2 v2;
5410 };
5411 
ci_parse_pplib_non_clock_info(struct radeon_device * rdev,struct radeon_ps * rps,struct _ATOM_PPLIB_NONCLOCK_INFO * non_clock_info,u8 table_rev)5412 static void ci_parse_pplib_non_clock_info(struct radeon_device *rdev,
5413 					  struct radeon_ps *rps,
5414 					  struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
5415 					  u8 table_rev)
5416 {
5417 	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
5418 	rps->class = le16_to_cpu(non_clock_info->usClassification);
5419 	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
5420 
5421 	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
5422 		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
5423 		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
5424 	} else {
5425 		rps->vclk = 0;
5426 		rps->dclk = 0;
5427 	}
5428 
5429 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
5430 		rdev->pm.dpm.boot_ps = rps;
5431 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
5432 		rdev->pm.dpm.uvd_ps = rps;
5433 }
5434 
ci_parse_pplib_clock_info(struct radeon_device * rdev,struct radeon_ps * rps,int index,union pplib_clock_info * clock_info)5435 static void ci_parse_pplib_clock_info(struct radeon_device *rdev,
5436 				      struct radeon_ps *rps, int index,
5437 				      union pplib_clock_info *clock_info)
5438 {
5439 	struct ci_power_info *pi = ci_get_pi(rdev);
5440 	struct ci_ps *ps = ci_get_ps(rps);
5441 	struct ci_pl *pl = &ps->performance_levels[index];
5442 
5443 	ps->performance_level_count = index + 1;
5444 
5445 	pl->sclk = le16_to_cpu(clock_info->ci.usEngineClockLow);
5446 	pl->sclk |= clock_info->ci.ucEngineClockHigh << 16;
5447 	pl->mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow);
5448 	pl->mclk |= clock_info->ci.ucMemoryClockHigh << 16;
5449 
5450 	pl->pcie_gen = r600_get_pcie_gen_support(rdev,
5451 						 pi->sys_pcie_mask,
5452 						 pi->vbios_boot_state.pcie_gen_bootup_value,
5453 						 clock_info->ci.ucPCIEGen);
5454 	pl->pcie_lane = r600_get_pcie_lane_support(rdev,
5455 						   pi->vbios_boot_state.pcie_lane_bootup_value,
5456 						   le16_to_cpu(clock_info->ci.usPCIELane));
5457 
5458 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
5459 		pi->acpi_pcie_gen = pl->pcie_gen;
5460 	}
5461 
5462 	if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
5463 		pi->ulv.supported = true;
5464 		pi->ulv.pl = *pl;
5465 		pi->ulv.cg_ulv_parameter = CISLANDS_CGULVPARAMETER_DFLT;
5466 	}
5467 
5468 	/* patch up boot state */
5469 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
5470 		pl->mclk = pi->vbios_boot_state.mclk_bootup_value;
5471 		pl->sclk = pi->vbios_boot_state.sclk_bootup_value;
5472 		pl->pcie_gen = pi->vbios_boot_state.pcie_gen_bootup_value;
5473 		pl->pcie_lane = pi->vbios_boot_state.pcie_lane_bootup_value;
5474 	}
5475 
5476 	switch (rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) {
5477 	case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY:
5478 		pi->use_pcie_powersaving_levels = true;
5479 		if (pi->pcie_gen_powersaving.max < pl->pcie_gen)
5480 			pi->pcie_gen_powersaving.max = pl->pcie_gen;
5481 		if (pi->pcie_gen_powersaving.min > pl->pcie_gen)
5482 			pi->pcie_gen_powersaving.min = pl->pcie_gen;
5483 		if (pi->pcie_lane_powersaving.max < pl->pcie_lane)
5484 			pi->pcie_lane_powersaving.max = pl->pcie_lane;
5485 		if (pi->pcie_lane_powersaving.min > pl->pcie_lane)
5486 			pi->pcie_lane_powersaving.min = pl->pcie_lane;
5487 		break;
5488 	case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE:
5489 		pi->use_pcie_performance_levels = true;
5490 		if (pi->pcie_gen_performance.max < pl->pcie_gen)
5491 			pi->pcie_gen_performance.max = pl->pcie_gen;
5492 		if (pi->pcie_gen_performance.min > pl->pcie_gen)
5493 			pi->pcie_gen_performance.min = pl->pcie_gen;
5494 		if (pi->pcie_lane_performance.max < pl->pcie_lane)
5495 			pi->pcie_lane_performance.max = pl->pcie_lane;
5496 		if (pi->pcie_lane_performance.min > pl->pcie_lane)
5497 			pi->pcie_lane_performance.min = pl->pcie_lane;
5498 		break;
5499 	default:
5500 		break;
5501 	}
5502 }
5503 
ci_parse_power_table(struct radeon_device * rdev)5504 static int ci_parse_power_table(struct radeon_device *rdev)
5505 {
5506 	struct radeon_mode_info *mode_info = &rdev->mode_info;
5507 	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
5508 	union pplib_power_state *power_state;
5509 	int i, j, k, non_clock_array_index, clock_array_index;
5510 	union pplib_clock_info *clock_info;
5511 	struct _StateArray *state_array;
5512 	struct _ClockInfoArray *clock_info_array;
5513 	struct _NonClockInfoArray *non_clock_info_array;
5514 	union power_info *power_info;
5515 	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
5516 	u16 data_offset;
5517 	u8 frev, crev;
5518 	u8 *power_state_offset;
5519 	struct ci_ps *ps;
5520 	int ret;
5521 
5522 	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
5523 				   &frev, &crev, &data_offset))
5524 		return -EINVAL;
5525 	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
5526 
5527 	state_array = (struct _StateArray *)
5528 		(mode_info->atom_context->bios + data_offset +
5529 		 le16_to_cpu(power_info->pplib.usStateArrayOffset));
5530 	clock_info_array = (struct _ClockInfoArray *)
5531 		(mode_info->atom_context->bios + data_offset +
5532 		 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
5533 	non_clock_info_array = (struct _NonClockInfoArray *)
5534 		(mode_info->atom_context->bios + data_offset +
5535 		 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
5536 
5537 	rdev->pm.dpm.ps = kcalloc(state_array->ucNumEntries,
5538 				  sizeof(struct radeon_ps),
5539 				  GFP_KERNEL);
5540 	if (!rdev->pm.dpm.ps)
5541 		return -ENOMEM;
5542 	power_state_offset = (u8 *)state_array->states;
5543 	rdev->pm.dpm.num_ps = 0;
5544 	for (i = 0; i < state_array->ucNumEntries; i++) {
5545 		u8 *idx;
5546 		power_state = (union pplib_power_state *)power_state_offset;
5547 		non_clock_array_index = power_state->v2.nonClockInfoIndex;
5548 		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
5549 			&non_clock_info_array->nonClockInfo[non_clock_array_index];
5550 		if (!rdev->pm.power_state[i].clock_info) {
5551 			ret = -EINVAL;
5552 			goto err_free_ps;
5553 		}
5554 		ps = kzalloc(sizeof(struct ci_ps), GFP_KERNEL);
5555 		if (ps == NULL) {
5556 			ret = -ENOMEM;
5557 			goto err_free_ps;
5558 		}
5559 		rdev->pm.dpm.ps[i].ps_priv = ps;
5560 		ci_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
5561 					      non_clock_info,
5562 					      non_clock_info_array->ucEntrySize);
5563 		k = 0;
5564 		idx = (u8 *)&power_state->v2.clockInfoIndex[0];
5565 		for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
5566 			clock_array_index = idx[j];
5567 			if (clock_array_index >= clock_info_array->ucNumEntries)
5568 				continue;
5569 			if (k >= CISLANDS_MAX_HARDWARE_POWERLEVELS)
5570 				break;
5571 			clock_info = (union pplib_clock_info *)
5572 				((u8 *)&clock_info_array->clockInfo[0] +
5573 				 (clock_array_index * clock_info_array->ucEntrySize));
5574 			ci_parse_pplib_clock_info(rdev,
5575 						  &rdev->pm.dpm.ps[i], k,
5576 						  clock_info);
5577 			k++;
5578 		}
5579 		power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
5580 		rdev->pm.dpm.num_ps = i + 1;
5581 	}
5582 
5583 	/* fill in the vce power states */
5584 	for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
5585 		u32 sclk, mclk;
5586 		clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
5587 		clock_info = (union pplib_clock_info *)
5588 			&clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
5589 		sclk = le16_to_cpu(clock_info->ci.usEngineClockLow);
5590 		sclk |= clock_info->ci.ucEngineClockHigh << 16;
5591 		mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow);
5592 		mclk |= clock_info->ci.ucMemoryClockHigh << 16;
5593 		rdev->pm.dpm.vce_states[i].sclk = sclk;
5594 		rdev->pm.dpm.vce_states[i].mclk = mclk;
5595 	}
5596 
5597 	return 0;
5598 
5599 err_free_ps:
5600 	for (i = 0; i < rdev->pm.dpm.num_ps; i++)
5601 		kfree(rdev->pm.dpm.ps[i].ps_priv);
5602 	kfree(rdev->pm.dpm.ps);
5603 	return ret;
5604 }
5605 
ci_get_vbios_boot_values(struct radeon_device * rdev,struct ci_vbios_boot_state * boot_state)5606 static int ci_get_vbios_boot_values(struct radeon_device *rdev,
5607 				    struct ci_vbios_boot_state *boot_state)
5608 {
5609 	struct radeon_mode_info *mode_info = &rdev->mode_info;
5610 	int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
5611 	ATOM_FIRMWARE_INFO_V2_2 *firmware_info;
5612 	u8 frev, crev;
5613 	u16 data_offset;
5614 
5615 	if (atom_parse_data_header(mode_info->atom_context, index, NULL,
5616 				   &frev, &crev, &data_offset)) {
5617 		firmware_info =
5618 			(ATOM_FIRMWARE_INFO_V2_2 *)(mode_info->atom_context->bios +
5619 						    data_offset);
5620 		boot_state->mvdd_bootup_value = le16_to_cpu(firmware_info->usBootUpMVDDCVoltage);
5621 		boot_state->vddc_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCVoltage);
5622 		boot_state->vddci_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCIVoltage);
5623 		boot_state->pcie_gen_bootup_value = ci_get_current_pcie_speed(rdev);
5624 		boot_state->pcie_lane_bootup_value = ci_get_current_pcie_lane_number(rdev);
5625 		boot_state->sclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultEngineClock);
5626 		boot_state->mclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultMemoryClock);
5627 
5628 		return 0;
5629 	}
5630 	return -EINVAL;
5631 }
5632 
ci_dpm_fini(struct radeon_device * rdev)5633 void ci_dpm_fini(struct radeon_device *rdev)
5634 {
5635 	int i;
5636 
5637 	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
5638 		kfree(rdev->pm.dpm.ps[i].ps_priv);
5639 	}
5640 	kfree(rdev->pm.dpm.ps);
5641 	kfree(rdev->pm.dpm.priv);
5642 	kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
5643 	r600_free_extended_power_table(rdev);
5644 }
5645 
ci_dpm_init(struct radeon_device * rdev)5646 int ci_dpm_init(struct radeon_device *rdev)
5647 {
5648 	int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
5649 	SMU7_Discrete_DpmTable  *dpm_table;
5650 	struct radeon_gpio_rec gpio;
5651 	u16 data_offset, size;
5652 	u8 frev, crev;
5653 	struct ci_power_info *pi;
5654 	enum pci_bus_speed speed_cap = PCI_SPEED_UNKNOWN;
5655 	struct pci_dev *root = rdev->pdev->bus->self;
5656 	int ret;
5657 
5658 	pi = kzalloc(sizeof(struct ci_power_info), GFP_KERNEL);
5659 	if (pi == NULL)
5660 		return -ENOMEM;
5661 	rdev->pm.dpm.priv = pi;
5662 
5663 	if (!pci_is_root_bus(rdev->pdev->bus))
5664 		speed_cap = pcie_get_speed_cap(root);
5665 	if (speed_cap == PCI_SPEED_UNKNOWN) {
5666 		pi->sys_pcie_mask = 0;
5667 	} else {
5668 		if (speed_cap == PCIE_SPEED_8_0GT)
5669 			pi->sys_pcie_mask = RADEON_PCIE_SPEED_25 |
5670 				RADEON_PCIE_SPEED_50 |
5671 				RADEON_PCIE_SPEED_80;
5672 		else if (speed_cap == PCIE_SPEED_5_0GT)
5673 			pi->sys_pcie_mask = RADEON_PCIE_SPEED_25 |
5674 				RADEON_PCIE_SPEED_50;
5675 		else
5676 			pi->sys_pcie_mask = RADEON_PCIE_SPEED_25;
5677 	}
5678 	pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID;
5679 
5680 	pi->pcie_gen_performance.max = RADEON_PCIE_GEN1;
5681 	pi->pcie_gen_performance.min = RADEON_PCIE_GEN3;
5682 	pi->pcie_gen_powersaving.max = RADEON_PCIE_GEN1;
5683 	pi->pcie_gen_powersaving.min = RADEON_PCIE_GEN3;
5684 
5685 	pi->pcie_lane_performance.max = 0;
5686 	pi->pcie_lane_performance.min = 16;
5687 	pi->pcie_lane_powersaving.max = 0;
5688 	pi->pcie_lane_powersaving.min = 16;
5689 
5690 	ret = ci_get_vbios_boot_values(rdev, &pi->vbios_boot_state);
5691 	if (ret) {
5692 		kfree(rdev->pm.dpm.priv);
5693 		return ret;
5694 	}
5695 
5696 	ret = r600_get_platform_caps(rdev);
5697 	if (ret) {
5698 		kfree(rdev->pm.dpm.priv);
5699 		return ret;
5700 	}
5701 
5702 	ret = r600_parse_extended_power_table(rdev);
5703 	if (ret) {
5704 		kfree(rdev->pm.dpm.priv);
5705 		return ret;
5706 	}
5707 
5708 	ret = ci_parse_power_table(rdev);
5709 	if (ret) {
5710 		kfree(rdev->pm.dpm.priv);
5711 		r600_free_extended_power_table(rdev);
5712 		return ret;
5713 	}
5714 
5715 	pi->dll_default_on = false;
5716 	pi->sram_end = SMC_RAM_END;
5717 
5718 	pi->activity_target[0] = CISLAND_TARGETACTIVITY_DFLT;
5719 	pi->activity_target[1] = CISLAND_TARGETACTIVITY_DFLT;
5720 	pi->activity_target[2] = CISLAND_TARGETACTIVITY_DFLT;
5721 	pi->activity_target[3] = CISLAND_TARGETACTIVITY_DFLT;
5722 	pi->activity_target[4] = CISLAND_TARGETACTIVITY_DFLT;
5723 	pi->activity_target[5] = CISLAND_TARGETACTIVITY_DFLT;
5724 	pi->activity_target[6] = CISLAND_TARGETACTIVITY_DFLT;
5725 	pi->activity_target[7] = CISLAND_TARGETACTIVITY_DFLT;
5726 
5727 	pi->mclk_activity_target = CISLAND_MCLK_TARGETACTIVITY_DFLT;
5728 
5729 	pi->sclk_dpm_key_disabled = 0;
5730 	pi->mclk_dpm_key_disabled = 0;
5731 	pi->pcie_dpm_key_disabled = 0;
5732 	pi->thermal_sclk_dpm_enabled = 0;
5733 
5734 	/* mclk dpm is unstable on some R7 260X cards with the old mc ucode */
5735 	if ((rdev->pdev->device == 0x6658) &&
5736 	    (rdev->mc_fw->size == (BONAIRE_MC_UCODE_SIZE * 4))) {
5737 		pi->mclk_dpm_key_disabled = 1;
5738 	}
5739 
5740 	pi->caps_sclk_ds = true;
5741 
5742 	pi->mclk_strobe_mode_threshold = 40000;
5743 	pi->mclk_stutter_mode_threshold = 40000;
5744 	pi->mclk_edc_enable_threshold = 40000;
5745 	pi->mclk_edc_wr_enable_threshold = 40000;
5746 
5747 	ci_initialize_powertune_defaults(rdev);
5748 
5749 	pi->caps_fps = false;
5750 
5751 	pi->caps_sclk_throttle_low_notification = false;
5752 
5753 	pi->caps_uvd_dpm = true;
5754 	pi->caps_vce_dpm = true;
5755 
5756 	ci_get_leakage_voltages(rdev);
5757 	ci_patch_dependency_tables_with_leakage(rdev);
5758 	ci_set_private_data_variables_based_on_pptable(rdev);
5759 
5760 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
5761 		kcalloc(4,
5762 			sizeof(struct radeon_clock_voltage_dependency_entry),
5763 			GFP_KERNEL);
5764 	if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
5765 		ci_dpm_fini(rdev);
5766 		return -ENOMEM;
5767 	}
5768 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
5769 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
5770 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
5771 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
5772 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
5773 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
5774 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
5775 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
5776 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
5777 
5778 	rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 4;
5779 	rdev->pm.dpm.dyn_state.sclk_mclk_delta = 15000;
5780 	rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
5781 
5782 	rdev->pm.dpm.dyn_state.valid_sclk_values.count = 0;
5783 	rdev->pm.dpm.dyn_state.valid_sclk_values.values = NULL;
5784 	rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
5785 	rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
5786 
5787 	if (rdev->family == CHIP_HAWAII) {
5788 		pi->thermal_temp_setting.temperature_low = 94500;
5789 		pi->thermal_temp_setting.temperature_high = 95000;
5790 		pi->thermal_temp_setting.temperature_shutdown = 104000;
5791 	} else {
5792 		pi->thermal_temp_setting.temperature_low = 99500;
5793 		pi->thermal_temp_setting.temperature_high = 100000;
5794 		pi->thermal_temp_setting.temperature_shutdown = 104000;
5795 	}
5796 
5797 	pi->uvd_enabled = false;
5798 
5799 	dpm_table = &pi->smc_state_table;
5800 
5801 	gpio = radeon_atombios_lookup_gpio(rdev, VDDC_VRHOT_GPIO_PINID);
5802 	if (gpio.valid) {
5803 		dpm_table->VRHotGpio = gpio.shift;
5804 		rdev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_REGULATOR_HOT;
5805 	} else {
5806 		dpm_table->VRHotGpio = CISLANDS_UNUSED_GPIO_PIN;
5807 		rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_REGULATOR_HOT;
5808 	}
5809 
5810 	gpio = radeon_atombios_lookup_gpio(rdev, PP_AC_DC_SWITCH_GPIO_PINID);
5811 	if (gpio.valid) {
5812 		dpm_table->AcDcGpio = gpio.shift;
5813 		rdev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_HARDWAREDC;
5814 	} else {
5815 		dpm_table->AcDcGpio = CISLANDS_UNUSED_GPIO_PIN;
5816 		rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_HARDWAREDC;
5817 	}
5818 
5819 	gpio = radeon_atombios_lookup_gpio(rdev, VDDC_PCC_GPIO_PINID);
5820 	if (gpio.valid) {
5821 		u32 tmp = RREG32_SMC(CNB_PWRMGT_CNTL);
5822 
5823 		switch (gpio.shift) {
5824 		case 0:
5825 			tmp &= ~GNB_SLOW_MODE_MASK;
5826 			tmp |= GNB_SLOW_MODE(1);
5827 			break;
5828 		case 1:
5829 			tmp &= ~GNB_SLOW_MODE_MASK;
5830 			tmp |= GNB_SLOW_MODE(2);
5831 			break;
5832 		case 2:
5833 			tmp |= GNB_SLOW;
5834 			break;
5835 		case 3:
5836 			tmp |= FORCE_NB_PS1;
5837 			break;
5838 		case 4:
5839 			tmp |= DPM_ENABLED;
5840 			break;
5841 		default:
5842 			DRM_DEBUG("Invalid PCC GPIO: %u!\n", gpio.shift);
5843 			break;
5844 		}
5845 		WREG32_SMC(CNB_PWRMGT_CNTL, tmp);
5846 	}
5847 
5848 	pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_NONE;
5849 	pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_NONE;
5850 	pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_NONE;
5851 	if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_GPIO_LUT))
5852 		pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO;
5853 	else if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2))
5854 		pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2;
5855 
5856 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL) {
5857 		if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT))
5858 			pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO;
5859 		else if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2))
5860 			pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2;
5861 		else
5862 			rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL;
5863 	}
5864 
5865 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_MVDDCONTROL) {
5866 		if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT))
5867 			pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO;
5868 		else if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_SVID2))
5869 			pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2;
5870 		else
5871 			rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_MVDDCONTROL;
5872 	}
5873 
5874 	pi->vddc_phase_shed_control = true;
5875 
5876 #if defined(CONFIG_ACPI)
5877 	pi->pcie_performance_request =
5878 		radeon_acpi_is_pcie_performance_request_supported(rdev);
5879 #else
5880 	pi->pcie_performance_request = false;
5881 #endif
5882 
5883 	if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
5884 				   &frev, &crev, &data_offset)) {
5885 		pi->caps_sclk_ss_support = true;
5886 		pi->caps_mclk_ss_support = true;
5887 		pi->dynamic_ss = true;
5888 	} else {
5889 		pi->caps_sclk_ss_support = false;
5890 		pi->caps_mclk_ss_support = false;
5891 		pi->dynamic_ss = true;
5892 	}
5893 
5894 	if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
5895 		pi->thermal_protection = true;
5896 	else
5897 		pi->thermal_protection = false;
5898 
5899 	pi->caps_dynamic_ac_timing = true;
5900 
5901 	pi->uvd_power_gated = false;
5902 
5903 	/* make sure dc limits are valid */
5904 	if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
5905 	    (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
5906 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
5907 			rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
5908 
5909 	pi->fan_ctrl_is_in_default_mode = true;
5910 
5911 	return 0;
5912 }
5913 
ci_dpm_debugfs_print_current_performance_level(struct radeon_device * rdev,struct seq_file * m)5914 void ci_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
5915 						    struct seq_file *m)
5916 {
5917 	struct ci_power_info *pi = ci_get_pi(rdev);
5918 	struct radeon_ps *rps = &pi->current_rps;
5919 	u32 sclk = ci_get_average_sclk_freq(rdev);
5920 	u32 mclk = ci_get_average_mclk_freq(rdev);
5921 
5922 	seq_printf(m, "uvd    %sabled\n", pi->uvd_enabled ? "en" : "dis");
5923 	seq_printf(m, "vce    %sabled\n", rps->vce_active ? "en" : "dis");
5924 	seq_printf(m, "power level avg    sclk: %u mclk: %u\n",
5925 		   sclk, mclk);
5926 }
5927 
ci_dpm_print_power_state(struct radeon_device * rdev,struct radeon_ps * rps)5928 void ci_dpm_print_power_state(struct radeon_device *rdev,
5929 			      struct radeon_ps *rps)
5930 {
5931 	struct ci_ps *ps = ci_get_ps(rps);
5932 	struct ci_pl *pl;
5933 	int i;
5934 
5935 	r600_dpm_print_class_info(rps->class, rps->class2);
5936 	r600_dpm_print_cap_info(rps->caps);
5937 	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
5938 	for (i = 0; i < ps->performance_level_count; i++) {
5939 		pl = &ps->performance_levels[i];
5940 		printk("\t\tpower level %d    sclk: %u mclk: %u pcie gen: %u pcie lanes: %u\n",
5941 		       i, pl->sclk, pl->mclk, pl->pcie_gen + 1, pl->pcie_lane);
5942 	}
5943 	r600_dpm_print_ps_status(rdev, rps);
5944 }
5945 
ci_dpm_get_current_sclk(struct radeon_device * rdev)5946 u32 ci_dpm_get_current_sclk(struct radeon_device *rdev)
5947 {
5948 	u32 sclk = ci_get_average_sclk_freq(rdev);
5949 
5950 	return sclk;
5951 }
5952 
ci_dpm_get_current_mclk(struct radeon_device * rdev)5953 u32 ci_dpm_get_current_mclk(struct radeon_device *rdev)
5954 {
5955 	u32 mclk = ci_get_average_mclk_freq(rdev);
5956 
5957 	return mclk;
5958 }
5959 
ci_dpm_get_sclk(struct radeon_device * rdev,bool low)5960 u32 ci_dpm_get_sclk(struct radeon_device *rdev, bool low)
5961 {
5962 	struct ci_power_info *pi = ci_get_pi(rdev);
5963 	struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
5964 
5965 	if (low)
5966 		return requested_state->performance_levels[0].sclk;
5967 	else
5968 		return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
5969 }
5970 
ci_dpm_get_mclk(struct radeon_device * rdev,bool low)5971 u32 ci_dpm_get_mclk(struct radeon_device *rdev, bool low)
5972 {
5973 	struct ci_power_info *pi = ci_get_pi(rdev);
5974 	struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
5975 
5976 	if (low)
5977 		return requested_state->performance_levels[0].mclk;
5978 	else
5979 		return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
5980 }
5981