1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include "amdgpu.h"
26 #include "amdgpu_sdma.h"
27 #include "amdgpu_ras.h"
28 
29 #define AMDGPU_CSA_SDMA_SIZE 64
30 /* SDMA CSA reside in the 3rd page of CSA */
31 #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
32 
33 /*
34  * GPU SDMA IP block helpers function.
35  */
36 
37 struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring)
38 {
39 	struct amdgpu_device *adev = ring->adev;
40 	int i;
41 
42 	for (i = 0; i < adev->sdma.num_instances; i++)
43 		if (ring == &adev->sdma.instance[i].ring ||
44 		    ring == &adev->sdma.instance[i].page)
45 			return &adev->sdma.instance[i];
46 
47 	return NULL;
48 }
49 
50 int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index)
51 {
52 	struct amdgpu_device *adev = ring->adev;
53 	int i;
54 
55 	for (i = 0; i < adev->sdma.num_instances; i++) {
56 		if (ring == &adev->sdma.instance[i].ring ||
57 			ring == &adev->sdma.instance[i].page) {
58 			*index = i;
59 			return 0;
60 		}
61 	}
62 
63 	return -EINVAL;
64 }
65 
66 uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring,
67 				     unsigned vmid)
68 {
69 	struct amdgpu_device *adev = ring->adev;
70 	uint64_t csa_mc_addr;
71 	uint32_t index = 0;
72 	int r;
73 
74 	/* don't enable OS preemption on SDMA under SRIOV */
75 	if (amdgpu_sriov_vf(adev) || vmid == 0 || !amdgpu_mcbp)
76 		return 0;
77 
78 	if (ring->is_mes_queue) {
79 		uint32_t offset = 0;
80 
81 		offset = offsetof(struct amdgpu_mes_ctx_meta_data,
82 				  sdma[ring->idx].sdma_meta_data);
83 		csa_mc_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);
84 	} else {
85 		r = amdgpu_sdma_get_index_from_ring(ring, &index);
86 
87 		if (r || index > 31)
88 			csa_mc_addr = 0;
89 		else
90 			csa_mc_addr = amdgpu_csa_vaddr(adev) +
91 				AMDGPU_CSA_SDMA_OFFSET +
92 				index * AMDGPU_CSA_SDMA_SIZE;
93 	}
94 
95 	return csa_mc_addr;
96 }
97 
98 int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,
99 			      struct ras_common_if *ras_block)
100 {
101 	int r, i;
102 
103 	r = amdgpu_ras_block_late_init(adev, ras_block);
104 	if (r)
105 		return r;
106 
107 	if (amdgpu_ras_is_supported(adev, ras_block->block)) {
108 		for (i = 0; i < adev->sdma.num_instances; i++) {
109 			r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq,
110 				AMDGPU_SDMA_IRQ_INSTANCE0 + i);
111 			if (r)
112 				goto late_fini;
113 		}
114 	}
115 
116 	return 0;
117 
118 late_fini:
119 	amdgpu_ras_block_late_fini(adev, ras_block);
120 	return r;
121 }
122 
123 int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,
124 		void *err_data,
125 		struct amdgpu_iv_entry *entry)
126 {
127 	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
128 
129 	if (amdgpu_sriov_vf(adev))
130 		return AMDGPU_RAS_SUCCESS;
131 
132 	amdgpu_ras_reset_gpu(adev);
133 
134 	return AMDGPU_RAS_SUCCESS;
135 }
136 
137 int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
138 				      struct amdgpu_irq_src *source,
139 				      struct amdgpu_iv_entry *entry)
140 {
141 	struct ras_common_if *ras_if = adev->sdma.ras_if;
142 	struct ras_dispatch_if ih_data = {
143 		.entry = entry,
144 	};
145 
146 	if (!ras_if)
147 		return 0;
148 
149 	ih_data.head = *ras_if;
150 
151 	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
152 	return 0;
153 }
154 
155 static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
156 {
157 	uint16_t version_major;
158 	const struct common_firmware_header *header = NULL;
159 	const struct sdma_firmware_header_v1_0 *hdr;
160 	const struct sdma_firmware_header_v2_0 *hdr_v2;
161 
162 	header = (const struct common_firmware_header *)
163 		sdma_inst->fw->data;
164 	version_major = le16_to_cpu(header->header_version_major);
165 
166 	switch (version_major) {
167 	case 1:
168 		hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
169 		sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
170 		sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
171 		break;
172 	case 2:
173 		hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;
174 		sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version);
175 		sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version);
176 		break;
177 	default:
178 		return -EINVAL;
179 	}
180 
181 	if (sdma_inst->feature_version >= 20)
182 		sdma_inst->burst_nop = true;
183 
184 	return 0;
185 }
186 
187 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
188 				  bool duplicate)
189 {
190 	int i;
191 
192 	for (i = 0; i < adev->sdma.num_instances; i++) {
193 		amdgpu_ucode_release(&adev->sdma.instance[i].fw);
194 		if (duplicate)
195 			break;
196 	}
197 
198 	memset((void *)adev->sdma.instance, 0,
199 	       sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
200 }
201 
202 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
203 			       u32 instance, bool duplicate)
204 {
205 	struct amdgpu_firmware_info *info = NULL;
206 	const struct common_firmware_header *header = NULL;
207 	int err, i;
208 	const struct sdma_firmware_header_v2_0 *sdma_hdr;
209 	uint16_t version_major;
210 	char ucode_prefix[30];
211 	char fw_name[40];
212 
213 	amdgpu_ucode_ip_version_decode(adev, SDMA0_HWIP, ucode_prefix, sizeof(ucode_prefix));
214 	if (instance == 0)
215 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s.bin", ucode_prefix);
216 	else
217 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s%d.bin", ucode_prefix, instance);
218 	err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw, fw_name);
219 	if (err)
220 		goto out;
221 
222 	header = (const struct common_firmware_header *)
223 		adev->sdma.instance[instance].fw->data;
224 	version_major = le16_to_cpu(header->header_version_major);
225 
226 	if ((duplicate && instance) || (!duplicate && version_major > 1)) {
227 		err = -EINVAL;
228 		goto out;
229 	}
230 
231 	err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);
232 	if (err)
233 		goto out;
234 
235 	if (duplicate) {
236 		for (i = 1; i < adev->sdma.num_instances; i++)
237 			memcpy((void *)&adev->sdma.instance[i],
238 			       (void *)&adev->sdma.instance[0],
239 			       sizeof(struct amdgpu_sdma_instance));
240 	}
241 
242 	if (amdgpu_sriov_vf(adev))
243 		return 0;
244 
245 	DRM_DEBUG("psp_load == '%s'\n",
246 		  adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
247 
248 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
249 		switch (version_major) {
250 		case 1:
251 			for (i = 0; i < adev->sdma.num_instances; i++) {
252 				if (!duplicate && (instance != i))
253 					continue;
254 				else {
255 					info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
256 					info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
257 					info->fw = adev->sdma.instance[i].fw;
258 					adev->firmware.fw_size +=
259 						ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
260 				}
261 			}
262 			break;
263 		case 2:
264 			sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
265 				adev->sdma.instance[0].fw->data;
266 			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];
267 			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;
268 			info->fw = adev->sdma.instance[0].fw;
269 			adev->firmware.fw_size +=
270 				ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);
271 			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];
272 			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;
273 			info->fw = adev->sdma.instance[0].fw;
274 			adev->firmware.fw_size +=
275 				ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
276 			break;
277 		default:
278 			err = -EINVAL;
279 		}
280 	}
281 
282 out:
283 	if (err)
284 		amdgpu_sdma_destroy_inst_ctx(adev, duplicate);
285 	return err;
286 }
287 
288 void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev)
289 {
290 	struct amdgpu_ring *sdma;
291 	int i;
292 
293 	for (i = 0; i < adev->sdma.num_instances; i++) {
294 		if (adev->sdma.has_page_queue) {
295 			sdma = &adev->sdma.instance[i].page;
296 			if (adev->mman.buffer_funcs_ring == sdma) {
297 				amdgpu_ttm_set_buffer_funcs_status(adev, false);
298 				break;
299 			}
300 		}
301 		sdma = &adev->sdma.instance[i].ring;
302 		if (adev->mman.buffer_funcs_ring == sdma) {
303 			amdgpu_ttm_set_buffer_funcs_status(adev, false);
304 			break;
305 		}
306 	}
307 }
308 
309 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev)
310 {
311 	int err = 0;
312 	struct amdgpu_sdma_ras *ras = NULL;
313 
314 	/* adev->sdma.ras is NULL, which means sdma does not
315 	 * support ras function, then do nothing here.
316 	 */
317 	if (!adev->sdma.ras)
318 		return 0;
319 
320 	ras = adev->sdma.ras;
321 
322 	err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
323 	if (err) {
324 		dev_err(adev->dev, "Failed to register sdma ras block!\n");
325 		return err;
326 	}
327 
328 	strcpy(ras->ras_block.ras_comm.name, "sdma");
329 	ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__SDMA;
330 	ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
331 	adev->sdma.ras_if = &ras->ras_block.ras_comm;
332 
333 	/* If not define special ras_late_init function, use default ras_late_init */
334 	if (!ras->ras_block.ras_late_init)
335 		ras->ras_block.ras_late_init = amdgpu_sdma_ras_late_init;
336 
337 	/* If not defined special ras_cb function, use default ras_cb */
338 	if (!ras->ras_block.ras_cb)
339 		ras->ras_block.ras_cb = amdgpu_sdma_process_ras_data_cb;
340 
341 	return 0;
342 }
343