1 /* 2 * QEMU PowerPC pSeries Logical Partition NUMA associativity handling 3 * 4 * Copyright IBM Corp. 2020 5 * 6 * Authors: 7 * Daniel Henrique Barboza <danielhb413@gmail.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or later. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include "qemu-common.h" 15 #include "hw/ppc/spapr_numa.h" 16 #include "hw/pci-host/spapr.h" 17 #include "hw/ppc/fdt.h" 18 19 /* Moved from hw/ppc/spapr_pci_nvlink2.c */ 20 #define SPAPR_GPU_NUMA_ID (cpu_to_be32(1)) 21 22 /* 23 * Retrieves max_dist_ref_points of the current NUMA affinity. 24 */ 25 static int get_max_dist_ref_points(SpaprMachineState *spapr) 26 { 27 return FORM1_DIST_REF_POINTS; 28 } 29 30 /* 31 * Retrieves numa_assoc_size of the current NUMA affinity. 32 */ 33 static int get_numa_assoc_size(SpaprMachineState *spapr) 34 { 35 return FORM1_NUMA_ASSOC_SIZE; 36 } 37 38 /* 39 * Retrieves vcpu_assoc_size of the current NUMA affinity. 40 * 41 * vcpu_assoc_size is the size of ibm,associativity array 42 * for CPUs, which has an extra element (vcpu_id) in the end. 43 */ 44 static int get_vcpu_assoc_size(SpaprMachineState *spapr) 45 { 46 return get_numa_assoc_size(spapr) + 1; 47 } 48 49 /* 50 * Retrieves the ibm,associativity array of NUMA node 'node_id' 51 * for the current NUMA affinity. 52 */ 53 static const uint32_t *get_associativity(SpaprMachineState *spapr, int node_id) 54 { 55 return spapr->FORM1_assoc_array[node_id]; 56 } 57 58 static bool spapr_numa_is_symmetrical(MachineState *ms) 59 { 60 int src, dst; 61 int nb_numa_nodes = ms->numa_state->num_nodes; 62 NodeInfo *numa_info = ms->numa_state->nodes; 63 64 for (src = 0; src < nb_numa_nodes; src++) { 65 for (dst = src; dst < nb_numa_nodes; dst++) { 66 if (numa_info[src].distance[dst] != 67 numa_info[dst].distance[src]) { 68 return false; 69 } 70 } 71 } 72 73 return true; 74 } 75 76 /* 77 * NVLink2-connected GPU RAM needs to be placed on a separate NUMA node. 78 * We assign a new numa ID per GPU in spapr_pci_collect_nvgpu() which is 79 * called from vPHB reset handler so we initialize the counter here. 80 * If no NUMA is configured from the QEMU side, we start from 1 as GPU RAM 81 * must be equally distant from any other node. 82 * The final value of spapr->gpu_numa_id is going to be written to 83 * max-associativity-domains in spapr_build_fdt(). 84 */ 85 unsigned int spapr_numa_initial_nvgpu_numa_id(MachineState *machine) 86 { 87 return MAX(1, machine->numa_state->num_nodes); 88 } 89 90 /* 91 * This function will translate the user distances into 92 * what the kernel understand as possible values: 10 93 * (local distance), 20, 40, 80 and 160, and return the equivalent 94 * NUMA level for each. Current heuristic is: 95 * - local distance (10) returns numa_level = 0x4, meaning there is 96 * no rounding for local distance 97 * - distances between 11 and 30 inclusive -> rounded to 20, 98 * numa_level = 0x3 99 * - distances between 31 and 60 inclusive -> rounded to 40, 100 * numa_level = 0x2 101 * - distances between 61 and 120 inclusive -> rounded to 80, 102 * numa_level = 0x1 103 * - everything above 120 returns numa_level = 0 to indicate that 104 * there is no match. This will be calculated as disntace = 160 105 * by the kernel (as of v5.9) 106 */ 107 static uint8_t spapr_numa_get_numa_level(uint8_t distance) 108 { 109 if (distance == 10) { 110 return 0x4; 111 } else if (distance > 11 && distance <= 30) { 112 return 0x3; 113 } else if (distance > 31 && distance <= 60) { 114 return 0x2; 115 } else if (distance > 61 && distance <= 120) { 116 return 0x1; 117 } 118 119 return 0; 120 } 121 122 static void spapr_numa_define_FORM1_domains(SpaprMachineState *spapr) 123 { 124 MachineState *ms = MACHINE(spapr); 125 NodeInfo *numa_info = ms->numa_state->nodes; 126 int nb_numa_nodes = ms->numa_state->num_nodes; 127 int src, dst, i, j; 128 129 /* 130 * Fill all associativity domains of non-zero NUMA nodes with 131 * node_id. This is required because the default value (0) is 132 * considered a match with associativity domains of node 0. 133 */ 134 for (i = 1; i < nb_numa_nodes; i++) { 135 for (j = 1; j < FORM1_DIST_REF_POINTS; j++) { 136 spapr->FORM1_assoc_array[i][j] = cpu_to_be32(i); 137 } 138 } 139 140 for (src = 0; src < nb_numa_nodes; src++) { 141 for (dst = src; dst < nb_numa_nodes; dst++) { 142 /* 143 * This is how the associativity domain between A and B 144 * is calculated: 145 * 146 * - get the distance D between them 147 * - get the correspondent NUMA level 'n_level' for D 148 * - all associativity arrays were initialized with their own 149 * numa_ids, and we're calculating the distance in node_id 150 * ascending order, starting from node id 0 (the first node 151 * retrieved by numa_state). This will have a cascade effect in 152 * the algorithm because the associativity domains that node 0 153 * defines will be carried over to other nodes, and node 1 154 * associativities will be carried over after taking node 0 155 * associativities into account, and so on. This happens because 156 * we'll assign assoc_src as the associativity domain of dst 157 * as well, for all NUMA levels beyond and including n_level. 158 * 159 * The PPC kernel expects the associativity domains of node 0 to 160 * be always 0, and this algorithm will grant that by default. 161 */ 162 uint8_t distance = numa_info[src].distance[dst]; 163 uint8_t n_level = spapr_numa_get_numa_level(distance); 164 uint32_t assoc_src; 165 166 /* 167 * n_level = 0 means that the distance is greater than our last 168 * rounded value (120). In this case there is no NUMA level match 169 * between src and dst and we can skip the remaining of the loop. 170 * 171 * The Linux kernel will assume that the distance between src and 172 * dst, in this case of no match, is 10 (local distance) doubled 173 * for each NUMA it didn't match. We have FORM1_DIST_REF_POINTS 174 * levels (4), so this gives us 10*2*2*2*2 = 160. 175 * 176 * This logic can be seen in the Linux kernel source code, as of 177 * v5.9, in arch/powerpc/mm/numa.c, function __node_distance(). 178 */ 179 if (n_level == 0) { 180 continue; 181 } 182 183 /* 184 * We must assign all assoc_src to dst, starting from n_level 185 * and going up to 0x1. 186 */ 187 for (i = n_level; i > 0; i--) { 188 assoc_src = spapr->FORM1_assoc_array[src][i]; 189 spapr->FORM1_assoc_array[dst][i] = assoc_src; 190 } 191 } 192 } 193 194 } 195 196 static void spapr_numa_FORM1_affinity_check(MachineState *machine) 197 { 198 int i; 199 200 /* 201 * Check we don't have a memory-less/cpu-less NUMA node 202 * Firmware relies on the existing memory/cpu topology to provide the 203 * NUMA topology to the kernel. 204 * And the linux kernel needs to know the NUMA topology at start 205 * to be able to hotplug CPUs later. 206 */ 207 if (machine->numa_state->num_nodes) { 208 for (i = 0; i < machine->numa_state->num_nodes; ++i) { 209 /* check for memory-less node */ 210 if (machine->numa_state->nodes[i].node_mem == 0) { 211 CPUState *cs; 212 int found = 0; 213 /* check for cpu-less node */ 214 CPU_FOREACH(cs) { 215 PowerPCCPU *cpu = POWERPC_CPU(cs); 216 if (cpu->node_id == i) { 217 found = 1; 218 break; 219 } 220 } 221 /* memory-less and cpu-less node */ 222 if (!found) { 223 error_report( 224 "Memory-less/cpu-less nodes are not supported with FORM1 NUMA (node %d)", i); 225 exit(EXIT_FAILURE); 226 } 227 } 228 } 229 } 230 231 if (!spapr_numa_is_symmetrical(machine)) { 232 error_report( 233 "Asymmetrical NUMA topologies aren't supported in the pSeries machine using FORM1 NUMA"); 234 exit(EXIT_FAILURE); 235 } 236 } 237 238 /* 239 * Set NUMA machine state data based on FORM1 affinity semantics. 240 */ 241 static void spapr_numa_FORM1_affinity_init(SpaprMachineState *spapr, 242 MachineState *machine) 243 { 244 SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr); 245 int nb_numa_nodes = machine->numa_state->num_nodes; 246 int i, j, max_nodes_with_gpus; 247 248 /* 249 * For all associativity arrays: first position is the size, 250 * position FORM1_DIST_REF_POINTS is always the numa_id, 251 * represented by the index 'i'. 252 * 253 * This will break on sparse NUMA setups, when/if QEMU starts 254 * to support it, because there will be no more guarantee that 255 * 'i' will be a valid node_id set by the user. 256 */ 257 for (i = 0; i < nb_numa_nodes; i++) { 258 spapr->FORM1_assoc_array[i][0] = cpu_to_be32(FORM1_DIST_REF_POINTS); 259 spapr->FORM1_assoc_array[i][FORM1_DIST_REF_POINTS] = cpu_to_be32(i); 260 } 261 262 /* 263 * Initialize NVLink GPU associativity arrays. We know that 264 * the first GPU will take the first available NUMA id, and 265 * we'll have a maximum of NVGPU_MAX_NUM GPUs in the machine. 266 * At this point we're not sure if there are GPUs or not, but 267 * let's initialize the associativity arrays and allow NVLink 268 * GPUs to be handled like regular NUMA nodes later on. 269 */ 270 max_nodes_with_gpus = nb_numa_nodes + NVGPU_MAX_NUM; 271 272 for (i = nb_numa_nodes; i < max_nodes_with_gpus; i++) { 273 spapr->FORM1_assoc_array[i][0] = cpu_to_be32(FORM1_DIST_REF_POINTS); 274 275 for (j = 1; j < FORM1_DIST_REF_POINTS; j++) { 276 uint32_t gpu_assoc = smc->pre_5_1_assoc_refpoints ? 277 SPAPR_GPU_NUMA_ID : cpu_to_be32(i); 278 spapr->FORM1_assoc_array[i][j] = gpu_assoc; 279 } 280 281 spapr->FORM1_assoc_array[i][FORM1_DIST_REF_POINTS] = cpu_to_be32(i); 282 } 283 284 /* 285 * Guests pseries-5.1 and older uses zeroed associativity domains, 286 * i.e. no domain definition based on NUMA distance input. 287 * 288 * Same thing with guests that have only one NUMA node. 289 */ 290 if (smc->pre_5_2_numa_associativity || 291 machine->numa_state->num_nodes <= 1) { 292 return; 293 } 294 295 spapr_numa_define_FORM1_domains(spapr); 296 } 297 298 void spapr_numa_associativity_init(SpaprMachineState *spapr, 299 MachineState *machine) 300 { 301 spapr_numa_FORM1_affinity_init(spapr, machine); 302 } 303 304 void spapr_numa_associativity_check(SpaprMachineState *spapr) 305 { 306 spapr_numa_FORM1_affinity_check(MACHINE(spapr)); 307 } 308 309 void spapr_numa_write_associativity_dt(SpaprMachineState *spapr, void *fdt, 310 int offset, int nodeid) 311 { 312 const uint32_t *associativity = get_associativity(spapr, nodeid); 313 314 _FDT((fdt_setprop(fdt, offset, "ibm,associativity", 315 associativity, 316 get_numa_assoc_size(spapr) * sizeof(uint32_t)))); 317 } 318 319 static uint32_t *spapr_numa_get_vcpu_assoc(SpaprMachineState *spapr, 320 PowerPCCPU *cpu) 321 { 322 const uint32_t *associativity = get_associativity(spapr, cpu->node_id); 323 int max_distance_ref_points = get_max_dist_ref_points(spapr); 324 int vcpu_assoc_size = get_vcpu_assoc_size(spapr); 325 uint32_t *vcpu_assoc = g_new(uint32_t, vcpu_assoc_size); 326 int index = spapr_get_vcpu_id(cpu); 327 328 /* 329 * VCPUs have an extra 'cpu_id' value in ibm,associativity 330 * compared to other resources. Increment the size at index 331 * 0, put cpu_id last, then copy the remaining associativity 332 * domains. 333 */ 334 vcpu_assoc[0] = cpu_to_be32(max_distance_ref_points + 1); 335 vcpu_assoc[vcpu_assoc_size - 1] = cpu_to_be32(index); 336 memcpy(vcpu_assoc + 1, associativity + 1, 337 (vcpu_assoc_size - 2) * sizeof(uint32_t)); 338 339 return vcpu_assoc; 340 } 341 342 int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt, 343 int offset, PowerPCCPU *cpu) 344 { 345 g_autofree uint32_t *vcpu_assoc = NULL; 346 int vcpu_assoc_size = get_vcpu_assoc_size(spapr); 347 348 vcpu_assoc = spapr_numa_get_vcpu_assoc(spapr, cpu); 349 350 /* Advertise NUMA via ibm,associativity */ 351 return fdt_setprop(fdt, offset, "ibm,associativity", vcpu_assoc, 352 vcpu_assoc_size * sizeof(uint32_t)); 353 } 354 355 356 int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt, 357 int offset) 358 { 359 MachineState *machine = MACHINE(spapr); 360 int max_distance_ref_points = get_max_dist_ref_points(spapr); 361 int nb_numa_nodes = machine->numa_state->num_nodes; 362 int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1; 363 uint32_t *int_buf, *cur_index, buf_len; 364 int ret, i; 365 366 /* ibm,associativity-lookup-arrays */ 367 buf_len = (nr_nodes * max_distance_ref_points + 2) * sizeof(uint32_t); 368 cur_index = int_buf = g_malloc0(buf_len); 369 int_buf[0] = cpu_to_be32(nr_nodes); 370 /* Number of entries per associativity list */ 371 int_buf[1] = cpu_to_be32(max_distance_ref_points); 372 cur_index += 2; 373 for (i = 0; i < nr_nodes; i++) { 374 /* 375 * For the lookup-array we use the ibm,associativity array of the 376 * current NUMA affinity, without the first element (size). 377 */ 378 const uint32_t *associativity = get_associativity(spapr, i); 379 memcpy(cur_index, ++associativity, 380 sizeof(uint32_t) * max_distance_ref_points); 381 cur_index += max_distance_ref_points; 382 } 383 ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf, 384 (cur_index - int_buf) * sizeof(uint32_t)); 385 g_free(int_buf); 386 387 return ret; 388 } 389 390 static void spapr_numa_FORM1_write_rtas_dt(SpaprMachineState *spapr, 391 void *fdt, int rtas) 392 { 393 MachineState *ms = MACHINE(spapr); 394 SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr); 395 uint32_t number_nvgpus_nodes = spapr->gpu_numa_id - 396 spapr_numa_initial_nvgpu_numa_id(ms); 397 uint32_t refpoints[] = { 398 cpu_to_be32(0x4), 399 cpu_to_be32(0x3), 400 cpu_to_be32(0x2), 401 cpu_to_be32(0x1), 402 }; 403 uint32_t nr_refpoints = ARRAY_SIZE(refpoints); 404 uint32_t maxdomain = ms->numa_state->num_nodes + number_nvgpus_nodes; 405 uint32_t maxdomains[] = { 406 cpu_to_be32(4), 407 cpu_to_be32(maxdomain), 408 cpu_to_be32(maxdomain), 409 cpu_to_be32(maxdomain), 410 cpu_to_be32(maxdomain) 411 }; 412 413 if (smc->pre_5_2_numa_associativity || 414 ms->numa_state->num_nodes <= 1) { 415 uint32_t legacy_refpoints[] = { 416 cpu_to_be32(0x4), 417 cpu_to_be32(0x4), 418 cpu_to_be32(0x2), 419 }; 420 uint32_t legacy_maxdomain = spapr->gpu_numa_id > 1 ? 1 : 0; 421 uint32_t legacy_maxdomains[] = { 422 cpu_to_be32(4), 423 cpu_to_be32(legacy_maxdomain), 424 cpu_to_be32(legacy_maxdomain), 425 cpu_to_be32(legacy_maxdomain), 426 cpu_to_be32(spapr->gpu_numa_id), 427 }; 428 429 G_STATIC_ASSERT(sizeof(legacy_refpoints) <= sizeof(refpoints)); 430 G_STATIC_ASSERT(sizeof(legacy_maxdomains) <= sizeof(maxdomains)); 431 432 nr_refpoints = 3; 433 434 memcpy(refpoints, legacy_refpoints, sizeof(legacy_refpoints)); 435 memcpy(maxdomains, legacy_maxdomains, sizeof(legacy_maxdomains)); 436 437 /* pseries-5.0 and older reference-points array is {0x4, 0x4} */ 438 if (smc->pre_5_1_assoc_refpoints) { 439 nr_refpoints = 2; 440 } 441 } 442 443 _FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points", 444 refpoints, nr_refpoints * sizeof(refpoints[0]))); 445 446 _FDT(fdt_setprop(fdt, rtas, "ibm,max-associativity-domains", 447 maxdomains, sizeof(maxdomains))); 448 } 449 450 /* 451 * Helper that writes ibm,associativity-reference-points and 452 * max-associativity-domains in the RTAS pointed by @rtas 453 * in the DT @fdt. 454 */ 455 void spapr_numa_write_rtas_dt(SpaprMachineState *spapr, void *fdt, int rtas) 456 { 457 spapr_numa_FORM1_write_rtas_dt(spapr, fdt, rtas); 458 } 459 460 static target_ulong h_home_node_associativity(PowerPCCPU *cpu, 461 SpaprMachineState *spapr, 462 target_ulong opcode, 463 target_ulong *args) 464 { 465 g_autofree uint32_t *vcpu_assoc = NULL; 466 target_ulong flags = args[0]; 467 target_ulong procno = args[1]; 468 PowerPCCPU *tcpu; 469 int idx, assoc_idx; 470 int vcpu_assoc_size = get_vcpu_assoc_size(spapr); 471 472 /* only support procno from H_REGISTER_VPA */ 473 if (flags != 0x1) { 474 return H_FUNCTION; 475 } 476 477 tcpu = spapr_find_cpu(procno); 478 if (tcpu == NULL) { 479 return H_P2; 480 } 481 482 /* 483 * Given that we want to be flexible with the sizes and indexes, 484 * we must consider that there is a hard limit of how many 485 * associativities domain we can fit in R4 up to R9, which would be 486 * 12 associativity domains for vcpus. Assert and bail if that's 487 * not the case. 488 */ 489 g_assert((vcpu_assoc_size - 1) <= 12); 490 491 vcpu_assoc = spapr_numa_get_vcpu_assoc(spapr, tcpu); 492 /* assoc_idx starts at 1 to skip associativity size */ 493 assoc_idx = 1; 494 495 #define ASSOCIATIVITY(a, b) (((uint64_t)(a) << 32) | \ 496 ((uint64_t)(b) & 0xffffffff)) 497 498 for (idx = 0; idx < 6; idx++) { 499 int32_t a, b; 500 501 /* 502 * vcpu_assoc[] will contain the associativity domains for tcpu, 503 * including tcpu->node_id and procno, meaning that we don't 504 * need to use these variables here. 505 * 506 * We'll read 2 values at a time to fill up the ASSOCIATIVITY() 507 * macro. The ternary will fill the remaining registers with -1 508 * after we went through vcpu_assoc[]. 509 */ 510 a = assoc_idx < vcpu_assoc_size ? 511 be32_to_cpu(vcpu_assoc[assoc_idx++]) : -1; 512 b = assoc_idx < vcpu_assoc_size ? 513 be32_to_cpu(vcpu_assoc[assoc_idx++]) : -1; 514 515 args[idx] = ASSOCIATIVITY(a, b); 516 } 517 #undef ASSOCIATIVITY 518 519 return H_SUCCESS; 520 } 521 522 static void spapr_numa_register_types(void) 523 { 524 /* Virtual Processor Home Node */ 525 spapr_register_hypercall(H_HOME_NODE_ASSOCIATIVITY, 526 h_home_node_associativity); 527 } 528 529 type_init(spapr_numa_register_types) 530