1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  *
23  */
24 #include <linux/list.h>
25 #include "amdgpu.h"
26 #include "amdgpu_xgmi.h"
27 #include "amdgpu_smu.h"
28 #include "df/df_3_6_offset.h"
29 
30 static DEFINE_MUTEX(xgmi_mutex);
31 
32 #define AMDGPU_MAX_XGMI_HIVE			8
33 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE		4
34 
35 static struct amdgpu_hive_info xgmi_hives[AMDGPU_MAX_XGMI_HIVE];
36 static unsigned hive_count = 0;
37 
38 void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)
39 {
40 	return &hive->device_list;
41 }
42 
43 /**
44  * DOC: AMDGPU XGMI Support
45  *
46  * XGMI is a high speed interconnect that joins multiple GPU cards
47  * into a homogeneous memory space that is organized by a collective
48  * hive ID and individual node IDs, both of which are 64-bit numbers.
49  *
50  * The file xgmi_device_id contains the unique per GPU device ID and
51  * is stored in the /sys/class/drm/card${cardno}/device/ directory.
52  *
53  * Inside the device directory a sub-directory 'xgmi_hive_info' is
54  * created which contains the hive ID and the list of nodes.
55  *
56  * The hive ID is stored in:
57  *   /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
58  *
59  * The node information is stored in numbered directories:
60  *   /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
61  *
62  * Each device has their own xgmi_hive_info direction with a mirror
63  * set of node sub-directories.
64  *
65  * The XGMI memory space is built by contiguously adding the power of
66  * two padded VRAM space from each node to each other.
67  *
68  */
69 
70 
71 static ssize_t amdgpu_xgmi_show_hive_id(struct device *dev,
72 		struct device_attribute *attr, char *buf)
73 {
74 	struct amdgpu_hive_info *hive =
75 			container_of(attr, struct amdgpu_hive_info, dev_attr);
76 
77 	return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);
78 }
79 
80 static int amdgpu_xgmi_sysfs_create(struct amdgpu_device *adev,
81 				    struct amdgpu_hive_info *hive)
82 {
83 	int ret = 0;
84 
85 	if (WARN_ON(hive->kobj))
86 		return -EINVAL;
87 
88 	hive->kobj = kobject_create_and_add("xgmi_hive_info", &adev->dev->kobj);
89 	if (!hive->kobj) {
90 		dev_err(adev->dev, "XGMI: Failed to allocate sysfs entry!\n");
91 		return -EINVAL;
92 	}
93 
94 	hive->dev_attr = (struct device_attribute) {
95 		.attr = {
96 			.name = "xgmi_hive_id",
97 			.mode = S_IRUGO,
98 
99 		},
100 		.show = amdgpu_xgmi_show_hive_id,
101 	};
102 
103 	ret = sysfs_create_file(hive->kobj, &hive->dev_attr.attr);
104 	if (ret) {
105 		dev_err(adev->dev, "XGMI: Failed to create device file xgmi_hive_id\n");
106 		kobject_del(hive->kobj);
107 		kobject_put(hive->kobj);
108 		hive->kobj = NULL;
109 	}
110 
111 	return ret;
112 }
113 
114 static void amdgpu_xgmi_sysfs_destroy(struct amdgpu_device *adev,
115 				    struct amdgpu_hive_info *hive)
116 {
117 	sysfs_remove_file(hive->kobj, &hive->dev_attr.attr);
118 	kobject_del(hive->kobj);
119 	kobject_put(hive->kobj);
120 	hive->kobj = NULL;
121 }
122 
123 static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
124 				     struct device_attribute *attr,
125 				     char *buf)
126 {
127 	struct drm_device *ddev = dev_get_drvdata(dev);
128 	struct amdgpu_device *adev = ddev->dev_private;
129 
130 	return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.xgmi.node_id);
131 
132 }
133 
134 #define AMDGPU_XGMI_SET_FICAA(o)	((o) | 0x456801)
135 static ssize_t amdgpu_xgmi_show_error(struct device *dev,
136 				      struct device_attribute *attr,
137 				      char *buf)
138 {
139 	struct drm_device *ddev = dev_get_drvdata(dev);
140 	struct amdgpu_device *adev = ddev->dev_private;
141 	uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;
142 	uint64_t fica_out;
143 	unsigned int error_count = 0;
144 
145 	ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);
146 	ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);
147 
148 	fica_out = adev->df_funcs->get_fica(adev, ficaa_pie_ctl_in);
149 	if (fica_out != 0x1f)
150 		pr_err("xGMI error counters not enabled!\n");
151 
152 	fica_out = adev->df_funcs->get_fica(adev, ficaa_pie_status_in);
153 
154 	if ((fica_out & 0xffff) == 2)
155 		error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);
156 
157 	adev->df_funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);
158 
159 	return snprintf(buf, PAGE_SIZE, "%d\n", error_count);
160 }
161 
162 
163 static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
164 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
165 
166 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
167 					 struct amdgpu_hive_info *hive)
168 {
169 	int ret = 0;
170 	char node[10] = { 0 };
171 
172 	/* Create xgmi device id file */
173 	ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);
174 	if (ret) {
175 		dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
176 		return ret;
177 	}
178 
179 	/* Create xgmi error file */
180 	ret = device_create_file(adev->dev, &dev_attr_xgmi_error);
181 	if (ret)
182 		pr_err("failed to create xgmi_error\n");
183 
184 
185 	/* Create sysfs link to hive info folder on the first device */
186 	if (adev != hive->adev) {
187 		ret = sysfs_create_link(&adev->dev->kobj, hive->kobj,
188 					"xgmi_hive_info");
189 		if (ret) {
190 			dev_err(adev->dev, "XGMI: Failed to create link to hive info");
191 			goto remove_file;
192 		}
193 	}
194 
195 	sprintf(node, "node%d", hive->number_devices);
196 	/* Create sysfs link form the hive folder to yourself */
197 	ret = sysfs_create_link(hive->kobj, &adev->dev->kobj, node);
198 	if (ret) {
199 		dev_err(adev->dev, "XGMI: Failed to create link from hive info");
200 		goto remove_link;
201 	}
202 
203 	goto success;
204 
205 
206 remove_link:
207 	sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique);
208 
209 remove_file:
210 	device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
211 
212 success:
213 	return ret;
214 }
215 
216 static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
217 					  struct amdgpu_hive_info *hive)
218 {
219 	device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
220 	sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique);
221 	sysfs_remove_link(hive->kobj, adev->ddev->unique);
222 }
223 
224 
225 
226 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock)
227 {
228 	int i;
229 	struct amdgpu_hive_info *tmp;
230 
231 	if (!adev->gmc.xgmi.hive_id)
232 		return NULL;
233 
234 	mutex_lock(&xgmi_mutex);
235 
236 	for (i = 0 ; i < hive_count; ++i) {
237 		tmp = &xgmi_hives[i];
238 		if (tmp->hive_id == adev->gmc.xgmi.hive_id) {
239 			if (lock)
240 				mutex_lock(&tmp->hive_lock);
241 			mutex_unlock(&xgmi_mutex);
242 			return tmp;
243 		}
244 	}
245 	if (i >= AMDGPU_MAX_XGMI_HIVE) {
246 		mutex_unlock(&xgmi_mutex);
247 		return NULL;
248 	}
249 
250 	/* initialize new hive if not exist */
251 	tmp = &xgmi_hives[hive_count++];
252 
253 	if (amdgpu_xgmi_sysfs_create(adev, tmp)) {
254 		mutex_unlock(&xgmi_mutex);
255 		return NULL;
256 	}
257 
258 	tmp->adev = adev;
259 	tmp->hive_id = adev->gmc.xgmi.hive_id;
260 	INIT_LIST_HEAD(&tmp->device_list);
261 	mutex_init(&tmp->hive_lock);
262 	mutex_init(&tmp->reset_lock);
263 
264 	if (lock)
265 		mutex_lock(&tmp->hive_lock);
266 	tmp->pstate = -1;
267 	mutex_unlock(&xgmi_mutex);
268 
269 	return tmp;
270 }
271 
272 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
273 {
274 	int ret = 0;
275 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0);
276 
277 	if (!hive)
278 		return 0;
279 
280 	if (hive->pstate == pstate)
281 		return 0;
282 
283 	dev_dbg(adev->dev, "Set xgmi pstate %d.\n", pstate);
284 
285 	if (is_support_sw_smu_xgmi(adev))
286 		ret = smu_set_xgmi_pstate(&adev->smu, pstate);
287 	if (ret)
288 		dev_err(adev->dev,
289 			"XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
290 			adev->gmc.xgmi.node_id,
291 			adev->gmc.xgmi.hive_id, ret);
292 
293 	return ret;
294 }
295 
296 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
297 {
298 	int ret = -EINVAL;
299 
300 	/* Each psp need to set the latest topology */
301 	ret = psp_xgmi_set_topology_info(&adev->psp,
302 					 hive->number_devices,
303 					 &adev->psp.xgmi_context.top_info);
304 	if (ret)
305 		dev_err(adev->dev,
306 			"XGMI: Set topology failure on device %llx, hive %llx, ret %d",
307 			adev->gmc.xgmi.node_id,
308 			adev->gmc.xgmi.hive_id, ret);
309 
310 	return ret;
311 }
312 
313 
314 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
315 		struct amdgpu_device *peer_adev)
316 {
317 	struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
318 	int i;
319 
320 	for (i = 0 ; i < top->num_nodes; ++i)
321 		if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
322 			return top->nodes[i].num_hops;
323 	return	-EINVAL;
324 }
325 
326 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
327 {
328 	struct psp_xgmi_topology_info *top_info;
329 	struct amdgpu_hive_info *hive;
330 	struct amdgpu_xgmi	*entry;
331 	struct amdgpu_device *tmp_adev = NULL;
332 
333 	int count = 0, ret = 0;
334 
335 	if (!adev->gmc.xgmi.supported)
336 		return 0;
337 
338 	if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
339 		ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id);
340 		if (ret) {
341 			dev_err(adev->dev,
342 				"XGMI: Failed to get hive id\n");
343 			return ret;
344 		}
345 
346 		ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id);
347 		if (ret) {
348 			dev_err(adev->dev,
349 				"XGMI: Failed to get node id\n");
350 			return ret;
351 		}
352 	} else {
353 		adev->gmc.xgmi.hive_id = 16;
354 		adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;
355 	}
356 
357 	hive = amdgpu_get_xgmi_hive(adev, 1);
358 	if (!hive) {
359 		ret = -EINVAL;
360 		dev_err(adev->dev,
361 			"XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",
362 			adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);
363 		goto exit;
364 	}
365 
366 	top_info = &adev->psp.xgmi_context.top_info;
367 
368 	list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);
369 	list_for_each_entry(entry, &hive->device_list, head)
370 		top_info->nodes[count++].node_id = entry->node_id;
371 	top_info->num_nodes = count;
372 	hive->number_devices = count;
373 
374 	if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
375 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
376 			/* update node list for other device in the hive */
377 			if (tmp_adev != adev) {
378 				top_info = &tmp_adev->psp.xgmi_context.top_info;
379 				top_info->nodes[count - 1].node_id =
380 					adev->gmc.xgmi.node_id;
381 				top_info->num_nodes = count;
382 			}
383 			ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
384 			if (ret)
385 				goto exit;
386 		}
387 
388 		/* get latest topology info for each device from psp */
389 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
390 			ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
391 					&tmp_adev->psp.xgmi_context.top_info);
392 			if (ret) {
393 				dev_err(tmp_adev->dev,
394 					"XGMI: Get topology failure on device %llx, hive %llx, ret %d",
395 					tmp_adev->gmc.xgmi.node_id,
396 					tmp_adev->gmc.xgmi.hive_id, ret);
397 				/* To do : continue with some node failed or disable the whole hive */
398 				goto exit;
399 			}
400 		}
401 	}
402 
403 	if (!ret)
404 		ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
405 
406 
407 	mutex_unlock(&hive->hive_lock);
408 exit:
409 	if (!ret)
410 		dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
411 			 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
412 	else
413 		dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
414 			adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
415 			ret);
416 
417 	return ret;
418 }
419 
420 void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
421 {
422 	struct amdgpu_hive_info *hive;
423 
424 	if (!adev->gmc.xgmi.supported)
425 		return;
426 
427 	hive = amdgpu_get_xgmi_hive(adev, 1);
428 	if (!hive)
429 		return;
430 
431 	if (!(hive->number_devices--)) {
432 		amdgpu_xgmi_sysfs_destroy(adev, hive);
433 		mutex_destroy(&hive->hive_lock);
434 		mutex_destroy(&hive->reset_lock);
435 	} else {
436 		amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
437 		mutex_unlock(&hive->hive_lock);
438 	}
439 }
440