xref: /openbmc/linux/drivers/gpu/drm/amd/amdkfd/kfd_crat.c (revision ae40e94f)
1 /*
2  * Copyright 2015-2017 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/pci.h>
24 #include <linux/acpi.h>
25 #include "kfd_crat.h"
26 #include "kfd_priv.h"
27 #include "kfd_topology.h"
28 #include "kfd_iommu.h"
29 #include "amdgpu_amdkfd.h"
30 
31 /* GPU Processor ID base for dGPUs for which VCRAT needs to be created.
32  * GPU processor ID are expressed with Bit[31]=1.
33  * The base is set to 0x8000_0000 + 0x1000 to avoid collision with GPU IDs
34  * used in the CRAT.
35  */
36 static uint32_t gpu_processor_id_low = 0x80001000;
37 
38 /* Return the next available gpu_processor_id and increment it for next GPU
39  *	@total_cu_count - Total CUs present in the GPU including ones
40  *			  masked off
41  */
42 static inline unsigned int get_and_inc_gpu_processor_id(
43 				unsigned int total_cu_count)
44 {
45 	int current_id = gpu_processor_id_low;
46 
47 	gpu_processor_id_low += total_cu_count;
48 	return current_id;
49 }
50 
51 /* Static table to describe GPU Cache information */
52 struct kfd_gpu_cache_info {
53 	uint32_t	cache_size;
54 	uint32_t	cache_level;
55 	uint32_t	flags;
56 	/* Indicates how many Compute Units share this cache
57 	 * Value = 1 indicates the cache is not shared
58 	 */
59 	uint32_t	num_cu_shared;
60 };
61 
62 static struct kfd_gpu_cache_info kaveri_cache_info[] = {
63 	{
64 		/* TCP L1 Cache per CU */
65 		.cache_size = 16,
66 		.cache_level = 1,
67 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
68 				CRAT_CACHE_FLAGS_DATA_CACHE |
69 				CRAT_CACHE_FLAGS_SIMD_CACHE),
70 		.num_cu_shared = 1,
71 
72 	},
73 	{
74 		/* Scalar L1 Instruction Cache (in SQC module) per bank */
75 		.cache_size = 16,
76 		.cache_level = 1,
77 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
78 				CRAT_CACHE_FLAGS_INST_CACHE |
79 				CRAT_CACHE_FLAGS_SIMD_CACHE),
80 		.num_cu_shared = 2,
81 	},
82 	{
83 		/* Scalar L1 Data Cache (in SQC module) per bank */
84 		.cache_size = 8,
85 		.cache_level = 1,
86 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
87 				CRAT_CACHE_FLAGS_DATA_CACHE |
88 				CRAT_CACHE_FLAGS_SIMD_CACHE),
89 		.num_cu_shared = 2,
90 	},
91 
92 	/* TODO: Add L2 Cache information */
93 };
94 
95 
96 static struct kfd_gpu_cache_info carrizo_cache_info[] = {
97 	{
98 		/* TCP L1 Cache per CU */
99 		.cache_size = 16,
100 		.cache_level = 1,
101 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
102 				CRAT_CACHE_FLAGS_DATA_CACHE |
103 				CRAT_CACHE_FLAGS_SIMD_CACHE),
104 		.num_cu_shared = 1,
105 	},
106 	{
107 		/* Scalar L1 Instruction Cache (in SQC module) per bank */
108 		.cache_size = 8,
109 		.cache_level = 1,
110 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
111 				CRAT_CACHE_FLAGS_INST_CACHE |
112 				CRAT_CACHE_FLAGS_SIMD_CACHE),
113 		.num_cu_shared = 4,
114 	},
115 	{
116 		/* Scalar L1 Data Cache (in SQC module) per bank. */
117 		.cache_size = 4,
118 		.cache_level = 1,
119 		.flags = (CRAT_CACHE_FLAGS_ENABLED |
120 				CRAT_CACHE_FLAGS_DATA_CACHE |
121 				CRAT_CACHE_FLAGS_SIMD_CACHE),
122 		.num_cu_shared = 4,
123 	},
124 
125 	/* TODO: Add L2 Cache information */
126 };
127 
128 /* NOTE: In future if more information is added to struct kfd_gpu_cache_info
129  * the following ASICs may need a separate table.
130  */
131 #define hawaii_cache_info kaveri_cache_info
132 #define tonga_cache_info carrizo_cache_info
133 #define fiji_cache_info  carrizo_cache_info
134 #define polaris10_cache_info carrizo_cache_info
135 #define polaris11_cache_info carrizo_cache_info
136 #define polaris12_cache_info carrizo_cache_info
137 /* TODO - check & update Vega10 cache details */
138 #define vega10_cache_info carrizo_cache_info
139 #define raven_cache_info carrizo_cache_info
140 
141 static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
142 		struct crat_subtype_computeunit *cu)
143 {
144 	dev->node_props.cpu_cores_count = cu->num_cpu_cores;
145 	dev->node_props.cpu_core_id_base = cu->processor_id_low;
146 	if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
147 		dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
148 
149 	pr_debug("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
150 			cu->processor_id_low);
151 }
152 
153 static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
154 		struct crat_subtype_computeunit *cu)
155 {
156 	dev->node_props.simd_id_base = cu->processor_id_low;
157 	dev->node_props.simd_count = cu->num_simd_cores;
158 	dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
159 	dev->node_props.max_waves_per_simd = cu->max_waves_simd;
160 	dev->node_props.wave_front_size = cu->wave_front_size;
161 	dev->node_props.array_count = cu->array_count;
162 	dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
163 	dev->node_props.simd_per_cu = cu->num_simd_per_cu;
164 	dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
165 	if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
166 		dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
167 	pr_debug("CU GPU: id_base=%d\n", cu->processor_id_low);
168 }
169 
170 /* kfd_parse_subtype_cu - parse compute unit subtypes and attach it to correct
171  * topology device present in the device_list
172  */
173 static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu,
174 				struct list_head *device_list)
175 {
176 	struct kfd_topology_device *dev;
177 
178 	pr_debug("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
179 			cu->proximity_domain, cu->hsa_capability);
180 	list_for_each_entry(dev, device_list, list) {
181 		if (cu->proximity_domain == dev->proximity_domain) {
182 			if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
183 				kfd_populated_cu_info_cpu(dev, cu);
184 
185 			if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
186 				kfd_populated_cu_info_gpu(dev, cu);
187 			break;
188 		}
189 	}
190 
191 	return 0;
192 }
193 
194 static struct kfd_mem_properties *
195 find_subtype_mem(uint32_t heap_type, uint32_t flags, uint32_t width,
196 		struct kfd_topology_device *dev)
197 {
198 	struct kfd_mem_properties *props;
199 
200 	list_for_each_entry(props, &dev->mem_props, list) {
201 		if (props->heap_type == heap_type
202 				&& props->flags == flags
203 				&& props->width == width)
204 			return props;
205 	}
206 
207 	return NULL;
208 }
209 /* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
210  * topology device present in the device_list
211  */
212 static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
213 				struct list_head *device_list)
214 {
215 	struct kfd_mem_properties *props;
216 	struct kfd_topology_device *dev;
217 	uint32_t heap_type;
218 	uint64_t size_in_bytes;
219 	uint32_t flags = 0;
220 	uint32_t width;
221 
222 	pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
223 			mem->proximity_domain);
224 	list_for_each_entry(dev, device_list, list) {
225 		if (mem->proximity_domain == dev->proximity_domain) {
226 			/* We're on GPU node */
227 			if (dev->node_props.cpu_cores_count == 0) {
228 				/* APU */
229 				if (mem->visibility_type == 0)
230 					heap_type =
231 						HSA_MEM_HEAP_TYPE_FB_PRIVATE;
232 				/* dGPU */
233 				else
234 					heap_type = mem->visibility_type;
235 			} else
236 				heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
237 
238 			if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
239 				flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
240 			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
241 				flags |= HSA_MEM_FLAGS_NON_VOLATILE;
242 
243 			size_in_bytes =
244 				((uint64_t)mem->length_high << 32) +
245 							mem->length_low;
246 			width = mem->width;
247 
248 			/* Multiple banks of the same type are aggregated into
249 			 * one. User mode doesn't care about multiple physical
250 			 * memory segments. It's managed as a single virtual
251 			 * heap for user mode.
252 			 */
253 			props = find_subtype_mem(heap_type, flags, width, dev);
254 			if (props) {
255 				props->size_in_bytes += size_in_bytes;
256 				break;
257 			}
258 
259 			props = kfd_alloc_struct(props);
260 			if (!props)
261 				return -ENOMEM;
262 
263 			props->heap_type = heap_type;
264 			props->flags = flags;
265 			props->size_in_bytes = size_in_bytes;
266 			props->width = width;
267 
268 			dev->node_props.mem_banks_count++;
269 			list_add_tail(&props->list, &dev->mem_props);
270 
271 			break;
272 		}
273 	}
274 
275 	return 0;
276 }
277 
278 /* kfd_parse_subtype_cache - parse cache subtypes and attach it to correct
279  * topology device present in the device_list
280  */
281 static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
282 			struct list_head *device_list)
283 {
284 	struct kfd_cache_properties *props;
285 	struct kfd_topology_device *dev;
286 	uint32_t id;
287 	uint32_t total_num_of_cu;
288 
289 	id = cache->processor_id_low;
290 
291 	pr_debug("Found cache entry in CRAT table with processor_id=%d\n", id);
292 	list_for_each_entry(dev, device_list, list) {
293 		total_num_of_cu = (dev->node_props.array_count *
294 					dev->node_props.cu_per_simd_array);
295 
296 		/* Cache infomration in CRAT doesn't have proximity_domain
297 		 * information as it is associated with a CPU core or GPU
298 		 * Compute Unit. So map the cache using CPU core Id or SIMD
299 		 * (GPU) ID.
300 		 * TODO: This works because currently we can safely assume that
301 		 *  Compute Units are parsed before caches are parsed. In
302 		 *  future, remove this dependency
303 		 */
304 		if ((id >= dev->node_props.cpu_core_id_base &&
305 			id <= dev->node_props.cpu_core_id_base +
306 				dev->node_props.cpu_cores_count) ||
307 			(id >= dev->node_props.simd_id_base &&
308 			id < dev->node_props.simd_id_base +
309 				total_num_of_cu)) {
310 			props = kfd_alloc_struct(props);
311 			if (!props)
312 				return -ENOMEM;
313 
314 			props->processor_id_low = id;
315 			props->cache_level = cache->cache_level;
316 			props->cache_size = cache->cache_size;
317 			props->cacheline_size = cache->cache_line_size;
318 			props->cachelines_per_tag = cache->lines_per_tag;
319 			props->cache_assoc = cache->associativity;
320 			props->cache_latency = cache->cache_latency;
321 			memcpy(props->sibling_map, cache->sibling_map,
322 					sizeof(props->sibling_map));
323 
324 			if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
325 				props->cache_type |= HSA_CACHE_TYPE_DATA;
326 			if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
327 				props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
328 			if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
329 				props->cache_type |= HSA_CACHE_TYPE_CPU;
330 			if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
331 				props->cache_type |= HSA_CACHE_TYPE_HSACU;
332 
333 			dev->cache_count++;
334 			dev->node_props.caches_count++;
335 			list_add_tail(&props->list, &dev->cache_props);
336 
337 			break;
338 		}
339 	}
340 
341 	return 0;
342 }
343 
344 /* kfd_parse_subtype_iolink - parse iolink subtypes and attach it to correct
345  * topology device present in the device_list
346  */
347 static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
348 					struct list_head *device_list)
349 {
350 	struct kfd_iolink_properties *props = NULL, *props2;
351 	struct kfd_topology_device *dev, *to_dev;
352 	uint32_t id_from;
353 	uint32_t id_to;
354 
355 	id_from = iolink->proximity_domain_from;
356 	id_to = iolink->proximity_domain_to;
357 
358 	pr_debug("Found IO link entry in CRAT table with id_from=%d, id_to %d\n",
359 			id_from, id_to);
360 	list_for_each_entry(dev, device_list, list) {
361 		if (id_from == dev->proximity_domain) {
362 			props = kfd_alloc_struct(props);
363 			if (!props)
364 				return -ENOMEM;
365 
366 			props->node_from = id_from;
367 			props->node_to = id_to;
368 			props->ver_maj = iolink->version_major;
369 			props->ver_min = iolink->version_minor;
370 			props->iolink_type = iolink->io_interface_type;
371 
372 			if (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
373 				props->weight = 20;
374 			else if (props->iolink_type == CRAT_IOLINK_TYPE_XGMI)
375 				props->weight = 15;
376 			else
377 				props->weight = node_distance(id_from, id_to);
378 
379 			props->min_latency = iolink->minimum_latency;
380 			props->max_latency = iolink->maximum_latency;
381 			props->min_bandwidth = iolink->minimum_bandwidth_mbs;
382 			props->max_bandwidth = iolink->maximum_bandwidth_mbs;
383 			props->rec_transfer_size =
384 					iolink->recommended_transfer_size;
385 
386 			dev->io_link_count++;
387 			dev->node_props.io_links_count++;
388 			list_add_tail(&props->list, &dev->io_link_props);
389 			break;
390 		}
391 	}
392 
393 	/* CPU topology is created before GPUs are detected, so CPU->GPU
394 	 * links are not built at that time. If a PCIe type is discovered, it
395 	 * means a GPU is detected and we are adding GPU->CPU to the topology.
396 	 * At this time, also add the corresponded CPU->GPU link if GPU
397 	 * is large bar.
398 	 * For xGMI, we only added the link with one direction in the crat
399 	 * table, add corresponded reversed direction link now.
400 	 */
401 	if (props && (iolink->flags & CRAT_IOLINK_FLAGS_BI_DIRECTIONAL)) {
402 		to_dev = kfd_topology_device_by_proximity_domain(id_to);
403 		if (!to_dev)
404 			return -ENODEV;
405 		/* same everything but the other direction */
406 		props2 = kmemdup(props, sizeof(*props2), GFP_KERNEL);
407 		props2->node_from = id_to;
408 		props2->node_to = id_from;
409 		props2->kobj = NULL;
410 		to_dev->io_link_count++;
411 		to_dev->node_props.io_links_count++;
412 		list_add_tail(&props2->list, &to_dev->io_link_props);
413 	}
414 
415 	return 0;
416 }
417 
418 /* kfd_parse_subtype - parse subtypes and attach it to correct topology device
419  * present in the device_list
420  *	@sub_type_hdr - subtype section of crat_image
421  *	@device_list - list of topology devices present in this crat_image
422  */
423 static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr,
424 				struct list_head *device_list)
425 {
426 	struct crat_subtype_computeunit *cu;
427 	struct crat_subtype_memory *mem;
428 	struct crat_subtype_cache *cache;
429 	struct crat_subtype_iolink *iolink;
430 	int ret = 0;
431 
432 	switch (sub_type_hdr->type) {
433 	case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
434 		cu = (struct crat_subtype_computeunit *)sub_type_hdr;
435 		ret = kfd_parse_subtype_cu(cu, device_list);
436 		break;
437 	case CRAT_SUBTYPE_MEMORY_AFFINITY:
438 		mem = (struct crat_subtype_memory *)sub_type_hdr;
439 		ret = kfd_parse_subtype_mem(mem, device_list);
440 		break;
441 	case CRAT_SUBTYPE_CACHE_AFFINITY:
442 		cache = (struct crat_subtype_cache *)sub_type_hdr;
443 		ret = kfd_parse_subtype_cache(cache, device_list);
444 		break;
445 	case CRAT_SUBTYPE_TLB_AFFINITY:
446 		/*
447 		 * For now, nothing to do here
448 		 */
449 		pr_debug("Found TLB entry in CRAT table (not processing)\n");
450 		break;
451 	case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
452 		/*
453 		 * For now, nothing to do here
454 		 */
455 		pr_debug("Found CCOMPUTE entry in CRAT table (not processing)\n");
456 		break;
457 	case CRAT_SUBTYPE_IOLINK_AFFINITY:
458 		iolink = (struct crat_subtype_iolink *)sub_type_hdr;
459 		ret = kfd_parse_subtype_iolink(iolink, device_list);
460 		break;
461 	default:
462 		pr_warn("Unknown subtype %d in CRAT\n",
463 				sub_type_hdr->type);
464 	}
465 
466 	return ret;
467 }
468 
469 /* kfd_parse_crat_table - parse CRAT table. For each node present in CRAT
470  * create a kfd_topology_device and add in to device_list. Also parse
471  * CRAT subtypes and attach it to appropriate kfd_topology_device
472  *	@crat_image - input image containing CRAT
473  *	@device_list - [OUT] list of kfd_topology_device generated after
474  *		       parsing crat_image
475  *	@proximity_domain - Proximity domain of the first device in the table
476  *
477  *	Return - 0 if successful else -ve value
478  */
479 int kfd_parse_crat_table(void *crat_image, struct list_head *device_list,
480 			 uint32_t proximity_domain)
481 {
482 	struct kfd_topology_device *top_dev = NULL;
483 	struct crat_subtype_generic *sub_type_hdr;
484 	uint16_t node_id;
485 	int ret = 0;
486 	struct crat_header *crat_table = (struct crat_header *)crat_image;
487 	uint16_t num_nodes;
488 	uint32_t image_len;
489 
490 	if (!crat_image)
491 		return -EINVAL;
492 
493 	if (!list_empty(device_list)) {
494 		pr_warn("Error device list should be empty\n");
495 		return -EINVAL;
496 	}
497 
498 	num_nodes = crat_table->num_domains;
499 	image_len = crat_table->length;
500 
501 	pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
502 
503 	for (node_id = 0; node_id < num_nodes; node_id++) {
504 		top_dev = kfd_create_topology_device(device_list);
505 		if (!top_dev)
506 			break;
507 		top_dev->proximity_domain = proximity_domain++;
508 	}
509 
510 	if (!top_dev) {
511 		ret = -ENOMEM;
512 		goto err;
513 	}
514 
515 	memcpy(top_dev->oem_id, crat_table->oem_id, CRAT_OEMID_LENGTH);
516 	memcpy(top_dev->oem_table_id, crat_table->oem_table_id,
517 			CRAT_OEMTABLEID_LENGTH);
518 	top_dev->oem_revision = crat_table->oem_revision;
519 
520 	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
521 	while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
522 			((char *)crat_image) + image_len) {
523 		if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
524 			ret = kfd_parse_subtype(sub_type_hdr, device_list);
525 			if (ret)
526 				break;
527 		}
528 
529 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
530 				sub_type_hdr->length);
531 	}
532 
533 err:
534 	if (ret)
535 		kfd_release_topology_device_list(device_list);
536 
537 	return ret;
538 }
539 
540 /* Helper function. See kfd_fill_gpu_cache_info for parameter description */
541 static int fill_in_pcache(struct crat_subtype_cache *pcache,
542 				struct kfd_gpu_cache_info *pcache_info,
543 				struct kfd_cu_info *cu_info,
544 				int mem_available,
545 				int cu_bitmask,
546 				int cache_type, unsigned int cu_processor_id,
547 				int cu_block)
548 {
549 	unsigned int cu_sibling_map_mask;
550 	int first_active_cu;
551 
552 	/* First check if enough memory is available */
553 	if (sizeof(struct crat_subtype_cache) > mem_available)
554 		return -ENOMEM;
555 
556 	cu_sibling_map_mask = cu_bitmask;
557 	cu_sibling_map_mask >>= cu_block;
558 	cu_sibling_map_mask &=
559 		((1 << pcache_info[cache_type].num_cu_shared) - 1);
560 	first_active_cu = ffs(cu_sibling_map_mask);
561 
562 	/* CU could be inactive. In case of shared cache find the first active
563 	 * CU. and incase of non-shared cache check if the CU is inactive. If
564 	 * inactive active skip it
565 	 */
566 	if (first_active_cu) {
567 		memset(pcache, 0, sizeof(struct crat_subtype_cache));
568 		pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
569 		pcache->length = sizeof(struct crat_subtype_cache);
570 		pcache->flags = pcache_info[cache_type].flags;
571 		pcache->processor_id_low = cu_processor_id
572 					 + (first_active_cu - 1);
573 		pcache->cache_level = pcache_info[cache_type].cache_level;
574 		pcache->cache_size = pcache_info[cache_type].cache_size;
575 
576 		/* Sibling map is w.r.t processor_id_low, so shift out
577 		 * inactive CU
578 		 */
579 		cu_sibling_map_mask =
580 			cu_sibling_map_mask >> (first_active_cu - 1);
581 
582 		pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF);
583 		pcache->sibling_map[1] =
584 				(uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
585 		pcache->sibling_map[2] =
586 				(uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
587 		pcache->sibling_map[3] =
588 				(uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
589 		return 0;
590 	}
591 	return 1;
592 }
593 
594 /* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info
595  * tables
596  *
597  *	@kdev - [IN] GPU device
598  *	@gpu_processor_id - [IN] GPU processor ID to which these caches
599  *			    associate
600  *	@available_size - [IN] Amount of memory available in pcache
601  *	@cu_info - [IN] Compute Unit info obtained from KGD
602  *	@pcache - [OUT] memory into which cache data is to be filled in.
603  *	@size_filled - [OUT] amount of data used up in pcache.
604  *	@num_of_entries - [OUT] number of caches added
605  */
606 static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
607 			int gpu_processor_id,
608 			int available_size,
609 			struct kfd_cu_info *cu_info,
610 			struct crat_subtype_cache *pcache,
611 			int *size_filled,
612 			int *num_of_entries)
613 {
614 	struct kfd_gpu_cache_info *pcache_info;
615 	int num_of_cache_types = 0;
616 	int i, j, k;
617 	int ct = 0;
618 	int mem_available = available_size;
619 	unsigned int cu_processor_id;
620 	int ret;
621 
622 	switch (kdev->device_info->asic_family) {
623 	case CHIP_KAVERI:
624 		pcache_info = kaveri_cache_info;
625 		num_of_cache_types = ARRAY_SIZE(kaveri_cache_info);
626 		break;
627 	case CHIP_HAWAII:
628 		pcache_info = hawaii_cache_info;
629 		num_of_cache_types = ARRAY_SIZE(hawaii_cache_info);
630 		break;
631 	case CHIP_CARRIZO:
632 		pcache_info = carrizo_cache_info;
633 		num_of_cache_types = ARRAY_SIZE(carrizo_cache_info);
634 		break;
635 	case CHIP_TONGA:
636 		pcache_info = tonga_cache_info;
637 		num_of_cache_types = ARRAY_SIZE(tonga_cache_info);
638 		break;
639 	case CHIP_FIJI:
640 		pcache_info = fiji_cache_info;
641 		num_of_cache_types = ARRAY_SIZE(fiji_cache_info);
642 		break;
643 	case CHIP_POLARIS10:
644 		pcache_info = polaris10_cache_info;
645 		num_of_cache_types = ARRAY_SIZE(polaris10_cache_info);
646 		break;
647 	case CHIP_POLARIS11:
648 		pcache_info = polaris11_cache_info;
649 		num_of_cache_types = ARRAY_SIZE(polaris11_cache_info);
650 		break;
651 	case CHIP_POLARIS12:
652 		pcache_info = polaris12_cache_info;
653 		num_of_cache_types = ARRAY_SIZE(polaris12_cache_info);
654 		break;
655 	case CHIP_VEGA10:
656 	case CHIP_VEGA12:
657 	case CHIP_VEGA20:
658 		pcache_info = vega10_cache_info;
659 		num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
660 		break;
661 	case CHIP_RAVEN:
662 		pcache_info = raven_cache_info;
663 		num_of_cache_types = ARRAY_SIZE(raven_cache_info);
664 		break;
665 	default:
666 		return -EINVAL;
667 	}
668 
669 	*size_filled = 0;
670 	*num_of_entries = 0;
671 
672 	/* For each type of cache listed in the kfd_gpu_cache_info table,
673 	 * go through all available Compute Units.
674 	 * The [i,j,k] loop will
675 	 *		if kfd_gpu_cache_info.num_cu_shared = 1
676 	 *			will parse through all available CU
677 	 *		If (kfd_gpu_cache_info.num_cu_shared != 1)
678 	 *			then it will consider only one CU from
679 	 *			the shared unit
680 	 */
681 
682 	for (ct = 0; ct < num_of_cache_types; ct++) {
683 		cu_processor_id = gpu_processor_id;
684 		for (i = 0; i < cu_info->num_shader_engines; i++) {
685 			for (j = 0; j < cu_info->num_shader_arrays_per_engine;
686 				j++) {
687 				for (k = 0; k < cu_info->num_cu_per_sh;
688 					k += pcache_info[ct].num_cu_shared) {
689 
690 					ret = fill_in_pcache(pcache,
691 						pcache_info,
692 						cu_info,
693 						mem_available,
694 						cu_info->cu_bitmap[i][j],
695 						ct,
696 						cu_processor_id,
697 						k);
698 
699 					if (ret < 0)
700 						break;
701 
702 					if (!ret) {
703 						pcache++;
704 						(*num_of_entries)++;
705 						mem_available -=
706 							sizeof(*pcache);
707 						(*size_filled) +=
708 							sizeof(*pcache);
709 					}
710 
711 					/* Move to next CU block */
712 					cu_processor_id +=
713 						pcache_info[ct].num_cu_shared;
714 				}
715 			}
716 		}
717 	}
718 
719 	pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
720 
721 	return 0;
722 }
723 
724 /*
725  * kfd_create_crat_image_acpi - Allocates memory for CRAT image and
726  * copies CRAT from ACPI (if available).
727  * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
728  *
729  *	@crat_image: CRAT read from ACPI. If no CRAT in ACPI then
730  *		     crat_image will be NULL
731  *	@size: [OUT] size of crat_image
732  *
733  *	Return 0 if successful else return error code
734  */
735 int kfd_create_crat_image_acpi(void **crat_image, size_t *size)
736 {
737 	struct acpi_table_header *crat_table;
738 	acpi_status status;
739 	void *pcrat_image;
740 
741 	if (!crat_image)
742 		return -EINVAL;
743 
744 	*crat_image = NULL;
745 
746 	/* Fetch the CRAT table from ACPI */
747 	status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
748 	if (status == AE_NOT_FOUND) {
749 		pr_warn("CRAT table not found\n");
750 		return -ENODATA;
751 	} else if (ACPI_FAILURE(status)) {
752 		const char *err = acpi_format_exception(status);
753 
754 		pr_err("CRAT table error: %s\n", err);
755 		return -EINVAL;
756 	}
757 
758 	if (ignore_crat) {
759 		pr_info("CRAT table disabled by module option\n");
760 		return -ENODATA;
761 	}
762 
763 	pcrat_image = kmemdup(crat_table, crat_table->length, GFP_KERNEL);
764 	if (!pcrat_image)
765 		return -ENOMEM;
766 
767 	*crat_image = pcrat_image;
768 	*size = crat_table->length;
769 
770 	return 0;
771 }
772 
773 /* Memory required to create Virtual CRAT.
774  * Since there is no easy way to predict the amount of memory required, the
775  * following amount are allocated for CPU and GPU Virtual CRAT. This is
776  * expected to cover all known conditions. But to be safe additional check
777  * is put in the code to ensure we don't overwrite.
778  */
779 #define VCRAT_SIZE_FOR_CPU	(2 * PAGE_SIZE)
780 #define VCRAT_SIZE_FOR_GPU	(3 * PAGE_SIZE)
781 
782 /* kfd_fill_cu_for_cpu - Fill in Compute info for the given CPU NUMA node
783  *
784  *	@numa_node_id: CPU NUMA node id
785  *	@avail_size: Available size in the memory
786  *	@sub_type_hdr: Memory into which compute info will be filled in
787  *
788  *	Return 0 if successful else return -ve value
789  */
790 static int kfd_fill_cu_for_cpu(int numa_node_id, int *avail_size,
791 				int proximity_domain,
792 				struct crat_subtype_computeunit *sub_type_hdr)
793 {
794 	const struct cpumask *cpumask;
795 
796 	*avail_size -= sizeof(struct crat_subtype_computeunit);
797 	if (*avail_size < 0)
798 		return -ENOMEM;
799 
800 	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
801 
802 	/* Fill in subtype header data */
803 	sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
804 	sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
805 	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
806 
807 	cpumask = cpumask_of_node(numa_node_id);
808 
809 	/* Fill in CU data */
810 	sub_type_hdr->flags |= CRAT_CU_FLAGS_CPU_PRESENT;
811 	sub_type_hdr->proximity_domain = proximity_domain;
812 	sub_type_hdr->processor_id_low = kfd_numa_node_to_apic_id(numa_node_id);
813 	if (sub_type_hdr->processor_id_low == -1)
814 		return -EINVAL;
815 
816 	sub_type_hdr->num_cpu_cores = cpumask_weight(cpumask);
817 
818 	return 0;
819 }
820 
821 /* kfd_fill_mem_info_for_cpu - Fill in Memory info for the given CPU NUMA node
822  *
823  *	@numa_node_id: CPU NUMA node id
824  *	@avail_size: Available size in the memory
825  *	@sub_type_hdr: Memory into which compute info will be filled in
826  *
827  *	Return 0 if successful else return -ve value
828  */
829 static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,
830 			int proximity_domain,
831 			struct crat_subtype_memory *sub_type_hdr)
832 {
833 	uint64_t mem_in_bytes = 0;
834 	pg_data_t *pgdat;
835 	int zone_type;
836 
837 	*avail_size -= sizeof(struct crat_subtype_memory);
838 	if (*avail_size < 0)
839 		return -ENOMEM;
840 
841 	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
842 
843 	/* Fill in subtype header data */
844 	sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
845 	sub_type_hdr->length = sizeof(struct crat_subtype_memory);
846 	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
847 
848 	/* Fill in Memory Subunit data */
849 
850 	/* Unlike si_meminfo, si_meminfo_node is not exported. So
851 	 * the following lines are duplicated from si_meminfo_node
852 	 * function
853 	 */
854 	pgdat = NODE_DATA(numa_node_id);
855 	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
856 		mem_in_bytes += zone_managed_pages(&pgdat->node_zones[zone_type]);
857 	mem_in_bytes <<= PAGE_SHIFT;
858 
859 	sub_type_hdr->length_low = lower_32_bits(mem_in_bytes);
860 	sub_type_hdr->length_high = upper_32_bits(mem_in_bytes);
861 	sub_type_hdr->proximity_domain = proximity_domain;
862 
863 	return 0;
864 }
865 
866 #ifdef CONFIG_X86_64
867 static int kfd_fill_iolink_info_for_cpu(int numa_node_id, int *avail_size,
868 				uint32_t *num_entries,
869 				struct crat_subtype_iolink *sub_type_hdr)
870 {
871 	int nid;
872 	struct cpuinfo_x86 *c = &cpu_data(0);
873 	uint8_t link_type;
874 
875 	if (c->x86_vendor == X86_VENDOR_AMD)
876 		link_type = CRAT_IOLINK_TYPE_HYPERTRANSPORT;
877 	else
878 		link_type = CRAT_IOLINK_TYPE_QPI_1_1;
879 
880 	*num_entries = 0;
881 
882 	/* Create IO links from this node to other CPU nodes */
883 	for_each_online_node(nid) {
884 		if (nid == numa_node_id) /* node itself */
885 			continue;
886 
887 		*avail_size -= sizeof(struct crat_subtype_iolink);
888 		if (*avail_size < 0)
889 			return -ENOMEM;
890 
891 		memset(sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
892 
893 		/* Fill in subtype header data */
894 		sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
895 		sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
896 		sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
897 
898 		/* Fill in IO link data */
899 		sub_type_hdr->proximity_domain_from = numa_node_id;
900 		sub_type_hdr->proximity_domain_to = nid;
901 		sub_type_hdr->io_interface_type = link_type;
902 
903 		(*num_entries)++;
904 		sub_type_hdr++;
905 	}
906 
907 	return 0;
908 }
909 #endif
910 
911 /* kfd_create_vcrat_image_cpu - Create Virtual CRAT for CPU
912  *
913  *	@pcrat_image: Fill in VCRAT for CPU
914  *	@size:	[IN] allocated size of crat_image.
915  *		[OUT] actual size of data filled in crat_image
916  */
917 static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
918 {
919 	struct crat_header *crat_table = (struct crat_header *)pcrat_image;
920 	struct acpi_table_header *acpi_table;
921 	acpi_status status;
922 	struct crat_subtype_generic *sub_type_hdr;
923 	int avail_size = *size;
924 	int numa_node_id;
925 #ifdef CONFIG_X86_64
926 	uint32_t entries = 0;
927 #endif
928 	int ret = 0;
929 
930 	if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_CPU)
931 		return -EINVAL;
932 
933 	/* Fill in CRAT Header.
934 	 * Modify length and total_entries as subunits are added.
935 	 */
936 	avail_size -= sizeof(struct crat_header);
937 	if (avail_size < 0)
938 		return -ENOMEM;
939 
940 	memset(crat_table, 0, sizeof(struct crat_header));
941 	memcpy(&crat_table->signature, CRAT_SIGNATURE,
942 			sizeof(crat_table->signature));
943 	crat_table->length = sizeof(struct crat_header);
944 
945 	status = acpi_get_table("DSDT", 0, &acpi_table);
946 	if (status != AE_OK)
947 		pr_warn("DSDT table not found for OEM information\n");
948 	else {
949 		crat_table->oem_revision = acpi_table->revision;
950 		memcpy(crat_table->oem_id, acpi_table->oem_id,
951 				CRAT_OEMID_LENGTH);
952 		memcpy(crat_table->oem_table_id, acpi_table->oem_table_id,
953 				CRAT_OEMTABLEID_LENGTH);
954 	}
955 	crat_table->total_entries = 0;
956 	crat_table->num_domains = 0;
957 
958 	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
959 
960 	for_each_online_node(numa_node_id) {
961 		if (kfd_numa_node_to_apic_id(numa_node_id) == -1)
962 			continue;
963 
964 		/* Fill in Subtype: Compute Unit */
965 		ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
966 			crat_table->num_domains,
967 			(struct crat_subtype_computeunit *)sub_type_hdr);
968 		if (ret < 0)
969 			return ret;
970 		crat_table->length += sub_type_hdr->length;
971 		crat_table->total_entries++;
972 
973 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
974 			sub_type_hdr->length);
975 
976 		/* Fill in Subtype: Memory */
977 		ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size,
978 			crat_table->num_domains,
979 			(struct crat_subtype_memory *)sub_type_hdr);
980 		if (ret < 0)
981 			return ret;
982 		crat_table->length += sub_type_hdr->length;
983 		crat_table->total_entries++;
984 
985 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
986 			sub_type_hdr->length);
987 
988 		/* Fill in Subtype: IO Link */
989 #ifdef CONFIG_X86_64
990 		ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
991 				&entries,
992 				(struct crat_subtype_iolink *)sub_type_hdr);
993 		if (ret < 0)
994 			return ret;
995 		crat_table->length += (sub_type_hdr->length * entries);
996 		crat_table->total_entries += entries;
997 
998 		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
999 				sub_type_hdr->length * entries);
1000 #else
1001 		pr_info("IO link not available for non x86 platforms\n");
1002 #endif
1003 
1004 		crat_table->num_domains++;
1005 	}
1006 
1007 	/* TODO: Add cache Subtype for CPU.
1008 	 * Currently, CPU cache information is available in function
1009 	 * detect_cache_attributes(cpu) defined in the file
1010 	 * ./arch/x86/kernel/cpu/intel_cacheinfo.c. This function is not
1011 	 * exported and to get the same information the code needs to be
1012 	 * duplicated.
1013 	 */
1014 
1015 	*size = crat_table->length;
1016 	pr_info("Virtual CRAT table created for CPU\n");
1017 
1018 	return 0;
1019 }
1020 
1021 static int kfd_fill_gpu_memory_affinity(int *avail_size,
1022 		struct kfd_dev *kdev, uint8_t type, uint64_t size,
1023 		struct crat_subtype_memory *sub_type_hdr,
1024 		uint32_t proximity_domain,
1025 		const struct kfd_local_mem_info *local_mem_info)
1026 {
1027 	*avail_size -= sizeof(struct crat_subtype_memory);
1028 	if (*avail_size < 0)
1029 		return -ENOMEM;
1030 
1031 	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
1032 	sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
1033 	sub_type_hdr->length = sizeof(struct crat_subtype_memory);
1034 	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
1035 
1036 	sub_type_hdr->proximity_domain = proximity_domain;
1037 
1038 	pr_debug("Fill gpu memory affinity - type 0x%x size 0x%llx\n",
1039 			type, size);
1040 
1041 	sub_type_hdr->length_low = lower_32_bits(size);
1042 	sub_type_hdr->length_high = upper_32_bits(size);
1043 
1044 	sub_type_hdr->width = local_mem_info->vram_width;
1045 	sub_type_hdr->visibility_type = type;
1046 
1047 	return 0;
1048 }
1049 
1050 /* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
1051  * to its NUMA node
1052  *	@avail_size: Available size in the memory
1053  *	@kdev - [IN] GPU device
1054  *	@sub_type_hdr: Memory into which io link info will be filled in
1055  *	@proximity_domain - proximity domain of the GPU node
1056  *
1057  *	Return 0 if successful else return -ve value
1058  */
1059 static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
1060 			struct kfd_dev *kdev,
1061 			struct crat_subtype_iolink *sub_type_hdr,
1062 			uint32_t proximity_domain)
1063 {
1064 	*avail_size -= sizeof(struct crat_subtype_iolink);
1065 	if (*avail_size < 0)
1066 		return -ENOMEM;
1067 
1068 	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
1069 
1070 	/* Fill in subtype header data */
1071 	sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
1072 	sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
1073 	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
1074 	if (kfd_dev_is_large_bar(kdev))
1075 		sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1076 
1077 	/* Fill in IOLINK subtype.
1078 	 * TODO: Fill-in other fields of iolink subtype
1079 	 */
1080 	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
1081 	sub_type_hdr->proximity_domain_from = proximity_domain;
1082 #ifdef CONFIG_NUMA
1083 	if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
1084 		sub_type_hdr->proximity_domain_to = 0;
1085 	else
1086 		sub_type_hdr->proximity_domain_to = kdev->pdev->dev.numa_node;
1087 #else
1088 	sub_type_hdr->proximity_domain_to = 0;
1089 #endif
1090 	return 0;
1091 }
1092 
1093 static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
1094 			struct kfd_dev *kdev,
1095 			struct crat_subtype_iolink *sub_type_hdr,
1096 			uint32_t proximity_domain_from,
1097 			uint32_t proximity_domain_to)
1098 {
1099 	*avail_size -= sizeof(struct crat_subtype_iolink);
1100 	if (*avail_size < 0)
1101 		return -ENOMEM;
1102 
1103 	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
1104 
1105 	sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
1106 	sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
1107 	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED |
1108 			       CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1109 
1110 	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
1111 	sub_type_hdr->proximity_domain_from = proximity_domain_from;
1112 	sub_type_hdr->proximity_domain_to = proximity_domain_to;
1113 	return 0;
1114 }
1115 
1116 /* kfd_create_vcrat_image_gpu - Create Virtual CRAT for CPU
1117  *
1118  *	@pcrat_image: Fill in VCRAT for GPU
1119  *	@size:	[IN] allocated size of crat_image.
1120  *		[OUT] actual size of data filled in crat_image
1121  */
1122 static int kfd_create_vcrat_image_gpu(void *pcrat_image,
1123 				      size_t *size, struct kfd_dev *kdev,
1124 				      uint32_t proximity_domain)
1125 {
1126 	struct crat_header *crat_table = (struct crat_header *)pcrat_image;
1127 	struct crat_subtype_generic *sub_type_hdr;
1128 	struct kfd_local_mem_info local_mem_info;
1129 	struct kfd_topology_device *peer_dev;
1130 	struct crat_subtype_computeunit *cu;
1131 	struct kfd_cu_info cu_info;
1132 	int avail_size = *size;
1133 	uint32_t total_num_of_cu;
1134 	int num_of_cache_entries = 0;
1135 	int cache_mem_filled = 0;
1136 	uint32_t nid = 0;
1137 	int ret = 0;
1138 
1139 	if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_GPU)
1140 		return -EINVAL;
1141 
1142 	/* Fill the CRAT Header.
1143 	 * Modify length and total_entries as subunits are added.
1144 	 */
1145 	avail_size -= sizeof(struct crat_header);
1146 	if (avail_size < 0)
1147 		return -ENOMEM;
1148 
1149 	memset(crat_table, 0, sizeof(struct crat_header));
1150 
1151 	memcpy(&crat_table->signature, CRAT_SIGNATURE,
1152 			sizeof(crat_table->signature));
1153 	/* Change length as we add more subtypes*/
1154 	crat_table->length = sizeof(struct crat_header);
1155 	crat_table->num_domains = 1;
1156 	crat_table->total_entries = 0;
1157 
1158 	/* Fill in Subtype: Compute Unit
1159 	 * First fill in the sub type header and then sub type data
1160 	 */
1161 	avail_size -= sizeof(struct crat_subtype_computeunit);
1162 	if (avail_size < 0)
1163 		return -ENOMEM;
1164 
1165 	sub_type_hdr = (struct crat_subtype_generic *)(crat_table + 1);
1166 	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
1167 
1168 	sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
1169 	sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
1170 	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
1171 
1172 	/* Fill CU subtype data */
1173 	cu = (struct crat_subtype_computeunit *)sub_type_hdr;
1174 	cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
1175 	cu->proximity_domain = proximity_domain;
1176 
1177 	amdgpu_amdkfd_get_cu_info(kdev->kgd, &cu_info);
1178 	cu->num_simd_per_cu = cu_info.simd_per_cu;
1179 	cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
1180 	cu->max_waves_simd = cu_info.max_waves_per_simd;
1181 
1182 	cu->wave_front_size = cu_info.wave_front_size;
1183 	cu->array_count = cu_info.num_shader_arrays_per_engine *
1184 		cu_info.num_shader_engines;
1185 	total_num_of_cu = (cu->array_count * cu_info.num_cu_per_sh);
1186 	cu->processor_id_low = get_and_inc_gpu_processor_id(total_num_of_cu);
1187 	cu->num_cu_per_array = cu_info.num_cu_per_sh;
1188 	cu->max_slots_scatch_cu = cu_info.max_scratch_slots_per_cu;
1189 	cu->num_banks = cu_info.num_shader_engines;
1190 	cu->lds_size_in_kb = cu_info.lds_size;
1191 
1192 	cu->hsa_capability = 0;
1193 
1194 	/* Check if this node supports IOMMU. During parsing this flag will
1195 	 * translate to HSA_CAP_ATS_PRESENT
1196 	 */
1197 	if (!kfd_iommu_check_device(kdev))
1198 		cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
1199 
1200 	crat_table->length += sub_type_hdr->length;
1201 	crat_table->total_entries++;
1202 
1203 	/* Fill in Subtype: Memory. Only on systems with large BAR (no
1204 	 * private FB), report memory as public. On other systems
1205 	 * report the total FB size (public+private) as a single
1206 	 * private heap.
1207 	 */
1208 	amdgpu_amdkfd_get_local_mem_info(kdev->kgd, &local_mem_info);
1209 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1210 			sub_type_hdr->length);
1211 
1212 	if (debug_largebar)
1213 		local_mem_info.local_mem_size_private = 0;
1214 
1215 	if (local_mem_info.local_mem_size_private == 0)
1216 		ret = kfd_fill_gpu_memory_affinity(&avail_size,
1217 				kdev, HSA_MEM_HEAP_TYPE_FB_PUBLIC,
1218 				local_mem_info.local_mem_size_public,
1219 				(struct crat_subtype_memory *)sub_type_hdr,
1220 				proximity_domain,
1221 				&local_mem_info);
1222 	else
1223 		ret = kfd_fill_gpu_memory_affinity(&avail_size,
1224 				kdev, HSA_MEM_HEAP_TYPE_FB_PRIVATE,
1225 				local_mem_info.local_mem_size_public +
1226 				local_mem_info.local_mem_size_private,
1227 				(struct crat_subtype_memory *)sub_type_hdr,
1228 				proximity_domain,
1229 				&local_mem_info);
1230 	if (ret < 0)
1231 		return ret;
1232 
1233 	crat_table->length += sizeof(struct crat_subtype_memory);
1234 	crat_table->total_entries++;
1235 
1236 	/* TODO: Fill in cache information. This information is NOT readily
1237 	 * available in KGD
1238 	 */
1239 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1240 		sub_type_hdr->length);
1241 	ret = kfd_fill_gpu_cache_info(kdev, cu->processor_id_low,
1242 				avail_size,
1243 				&cu_info,
1244 				(struct crat_subtype_cache *)sub_type_hdr,
1245 				&cache_mem_filled,
1246 				&num_of_cache_entries);
1247 
1248 	if (ret < 0)
1249 		return ret;
1250 
1251 	crat_table->length += cache_mem_filled;
1252 	crat_table->total_entries += num_of_cache_entries;
1253 	avail_size -= cache_mem_filled;
1254 
1255 	/* Fill in Subtype: IO_LINKS
1256 	 *  Only direct links are added here which is Link from GPU to
1257 	 *  to its NUMA node. Indirect links are added by userspace.
1258 	 */
1259 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
1260 		cache_mem_filled);
1261 	ret = kfd_fill_gpu_direct_io_link_to_cpu(&avail_size, kdev,
1262 		(struct crat_subtype_iolink *)sub_type_hdr, proximity_domain);
1263 
1264 	if (ret < 0)
1265 		return ret;
1266 
1267 	crat_table->length += sub_type_hdr->length;
1268 	crat_table->total_entries++;
1269 
1270 
1271 	/* Fill in Subtype: IO_LINKS
1272 	 * Direct links from GPU to other GPUs through xGMI.
1273 	 * We will loop GPUs that already be processed (with lower value
1274 	 * of proximity_domain), add the link for the GPUs with same
1275 	 * hive id (from this GPU to other GPU) . The reversed iolink
1276 	 * (from other GPU to this GPU) will be added
1277 	 * in kfd_parse_subtype_iolink.
1278 	 */
1279 	if (kdev->hive_id) {
1280 		for (nid = 0; nid < proximity_domain; ++nid) {
1281 			peer_dev = kfd_topology_device_by_proximity_domain(nid);
1282 			if (!peer_dev->gpu)
1283 				continue;
1284 			if (peer_dev->gpu->hive_id != kdev->hive_id)
1285 				continue;
1286 			sub_type_hdr = (typeof(sub_type_hdr))(
1287 				(char *)sub_type_hdr +
1288 				sizeof(struct crat_subtype_iolink));
1289 			ret = kfd_fill_gpu_xgmi_link_to_gpu(
1290 				&avail_size, kdev,
1291 				(struct crat_subtype_iolink *)sub_type_hdr,
1292 				proximity_domain, nid);
1293 			if (ret < 0)
1294 				return ret;
1295 			crat_table->length += sub_type_hdr->length;
1296 			crat_table->total_entries++;
1297 		}
1298 	}
1299 	*size = crat_table->length;
1300 	pr_info("Virtual CRAT table created for GPU\n");
1301 
1302 	return ret;
1303 }
1304 
1305 /* kfd_create_crat_image_virtual - Allocates memory for CRAT image and
1306  *		creates a Virtual CRAT (VCRAT) image
1307  *
1308  * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
1309  *
1310  *	@crat_image: VCRAT image created because ACPI does not have a
1311  *		     CRAT for this device
1312  *	@size: [OUT] size of virtual crat_image
1313  *	@flags:	COMPUTE_UNIT_CPU - Create VCRAT for CPU device
1314  *		COMPUTE_UNIT_GPU - Create VCRAT for GPU
1315  *		(COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU) - Create VCRAT for APU
1316  *			-- this option is not currently implemented.
1317  *			The assumption is that all AMD APUs will have CRAT
1318  *	@kdev: Valid kfd_device required if flags contain COMPUTE_UNIT_GPU
1319  *
1320  *	Return 0 if successful else return -ve value
1321  */
1322 int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
1323 				  int flags, struct kfd_dev *kdev,
1324 				  uint32_t proximity_domain)
1325 {
1326 	void *pcrat_image = NULL;
1327 	int ret = 0;
1328 
1329 	if (!crat_image)
1330 		return -EINVAL;
1331 
1332 	*crat_image = NULL;
1333 
1334 	/* Allocate one VCRAT_SIZE_FOR_CPU for CPU virtual CRAT image and
1335 	 * VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image. This should cover
1336 	 * all the current conditions. A check is put not to overwrite beyond
1337 	 * allocated size
1338 	 */
1339 	switch (flags) {
1340 	case COMPUTE_UNIT_CPU:
1341 		pcrat_image = kmalloc(VCRAT_SIZE_FOR_CPU, GFP_KERNEL);
1342 		if (!pcrat_image)
1343 			return -ENOMEM;
1344 		*size = VCRAT_SIZE_FOR_CPU;
1345 		ret = kfd_create_vcrat_image_cpu(pcrat_image, size);
1346 		break;
1347 	case COMPUTE_UNIT_GPU:
1348 		if (!kdev)
1349 			return -EINVAL;
1350 		pcrat_image = kmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
1351 		if (!pcrat_image)
1352 			return -ENOMEM;
1353 		*size = VCRAT_SIZE_FOR_GPU;
1354 		ret = kfd_create_vcrat_image_gpu(pcrat_image, size, kdev,
1355 						 proximity_domain);
1356 		break;
1357 	case (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU):
1358 		/* TODO: */
1359 		ret = -EINVAL;
1360 		pr_err("VCRAT not implemented for APU\n");
1361 		break;
1362 	default:
1363 		ret = -EINVAL;
1364 	}
1365 
1366 	if (!ret)
1367 		*crat_image = pcrat_image;
1368 	else
1369 		kfree(pcrat_image);
1370 
1371 	return ret;
1372 }
1373 
1374 
1375 /* kfd_destroy_crat_image
1376  *
1377  *	@crat_image: [IN] - crat_image from kfd_create_crat_image_xxx(..)
1378  *
1379  */
1380 void kfd_destroy_crat_image(void *crat_image)
1381 {
1382 	kfree(crat_image);
1383 }
1384