1 /* 2 * QEMU PowerPC pSeries Logical Partition NUMA associativity handling 3 * 4 * Copyright IBM Corp. 2020 5 * 6 * Authors: 7 * Daniel Henrique Barboza <danielhb413@gmail.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or later. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include "qemu-common.h" 15 #include "hw/ppc/spapr_numa.h" 16 #include "hw/pci-host/spapr.h" 17 #include "hw/ppc/fdt.h" 18 19 /* Moved from hw/ppc/spapr_pci_nvlink2.c */ 20 #define SPAPR_GPU_NUMA_ID (cpu_to_be32(1)) 21 22 /* 23 * Retrieves max_dist_ref_points of the current NUMA affinity. 24 */ 25 static int get_max_dist_ref_points(SpaprMachineState *spapr) 26 { 27 return FORM1_DIST_REF_POINTS; 28 } 29 30 /* 31 * Retrieves numa_assoc_size of the current NUMA affinity. 32 */ 33 static int get_numa_assoc_size(SpaprMachineState *spapr) 34 { 35 return FORM1_NUMA_ASSOC_SIZE; 36 } 37 38 /* 39 * Retrieves vcpu_assoc_size of the current NUMA affinity. 40 * 41 * vcpu_assoc_size is the size of ibm,associativity array 42 * for CPUs, which has an extra element (vcpu_id) in the end. 43 */ 44 static int get_vcpu_assoc_size(SpaprMachineState *spapr) 45 { 46 return get_numa_assoc_size(spapr) + 1; 47 } 48 49 static bool spapr_numa_is_symmetrical(MachineState *ms) 50 { 51 int src, dst; 52 int nb_numa_nodes = ms->numa_state->num_nodes; 53 NodeInfo *numa_info = ms->numa_state->nodes; 54 55 for (src = 0; src < nb_numa_nodes; src++) { 56 for (dst = src; dst < nb_numa_nodes; dst++) { 57 if (numa_info[src].distance[dst] != 58 numa_info[dst].distance[src]) { 59 return false; 60 } 61 } 62 } 63 64 return true; 65 } 66 67 /* 68 * NVLink2-connected GPU RAM needs to be placed on a separate NUMA node. 69 * We assign a new numa ID per GPU in spapr_pci_collect_nvgpu() which is 70 * called from vPHB reset handler so we initialize the counter here. 71 * If no NUMA is configured from the QEMU side, we start from 1 as GPU RAM 72 * must be equally distant from any other node. 73 * The final value of spapr->gpu_numa_id is going to be written to 74 * max-associativity-domains in spapr_build_fdt(). 75 */ 76 unsigned int spapr_numa_initial_nvgpu_numa_id(MachineState *machine) 77 { 78 return MAX(1, machine->numa_state->num_nodes); 79 } 80 81 /* 82 * This function will translate the user distances into 83 * what the kernel understand as possible values: 10 84 * (local distance), 20, 40, 80 and 160, and return the equivalent 85 * NUMA level for each. Current heuristic is: 86 * - local distance (10) returns numa_level = 0x4, meaning there is 87 * no rounding for local distance 88 * - distances between 11 and 30 inclusive -> rounded to 20, 89 * numa_level = 0x3 90 * - distances between 31 and 60 inclusive -> rounded to 40, 91 * numa_level = 0x2 92 * - distances between 61 and 120 inclusive -> rounded to 80, 93 * numa_level = 0x1 94 * - everything above 120 returns numa_level = 0 to indicate that 95 * there is no match. This will be calculated as disntace = 160 96 * by the kernel (as of v5.9) 97 */ 98 static uint8_t spapr_numa_get_numa_level(uint8_t distance) 99 { 100 if (distance == 10) { 101 return 0x4; 102 } else if (distance > 11 && distance <= 30) { 103 return 0x3; 104 } else if (distance > 31 && distance <= 60) { 105 return 0x2; 106 } else if (distance > 61 && distance <= 120) { 107 return 0x1; 108 } 109 110 return 0; 111 } 112 113 static void spapr_numa_define_FORM1_domains(SpaprMachineState *spapr) 114 { 115 MachineState *ms = MACHINE(spapr); 116 NodeInfo *numa_info = ms->numa_state->nodes; 117 int nb_numa_nodes = ms->numa_state->num_nodes; 118 int src, dst, i, j; 119 120 /* 121 * Fill all associativity domains of non-zero NUMA nodes with 122 * node_id. This is required because the default value (0) is 123 * considered a match with associativity domains of node 0. 124 */ 125 for (i = 1; i < nb_numa_nodes; i++) { 126 for (j = 1; j < FORM1_DIST_REF_POINTS; j++) { 127 spapr->numa_assoc_array[i][j] = cpu_to_be32(i); 128 } 129 } 130 131 for (src = 0; src < nb_numa_nodes; src++) { 132 for (dst = src; dst < nb_numa_nodes; dst++) { 133 /* 134 * This is how the associativity domain between A and B 135 * is calculated: 136 * 137 * - get the distance D between them 138 * - get the correspondent NUMA level 'n_level' for D 139 * - all associativity arrays were initialized with their own 140 * numa_ids, and we're calculating the distance in node_id 141 * ascending order, starting from node id 0 (the first node 142 * retrieved by numa_state). This will have a cascade effect in 143 * the algorithm because the associativity domains that node 0 144 * defines will be carried over to other nodes, and node 1 145 * associativities will be carried over after taking node 0 146 * associativities into account, and so on. This happens because 147 * we'll assign assoc_src as the associativity domain of dst 148 * as well, for all NUMA levels beyond and including n_level. 149 * 150 * The PPC kernel expects the associativity domains of node 0 to 151 * be always 0, and this algorithm will grant that by default. 152 */ 153 uint8_t distance = numa_info[src].distance[dst]; 154 uint8_t n_level = spapr_numa_get_numa_level(distance); 155 uint32_t assoc_src; 156 157 /* 158 * n_level = 0 means that the distance is greater than our last 159 * rounded value (120). In this case there is no NUMA level match 160 * between src and dst and we can skip the remaining of the loop. 161 * 162 * The Linux kernel will assume that the distance between src and 163 * dst, in this case of no match, is 10 (local distance) doubled 164 * for each NUMA it didn't match. We have FORM1_DIST_REF_POINTS 165 * levels (4), so this gives us 10*2*2*2*2 = 160. 166 * 167 * This logic can be seen in the Linux kernel source code, as of 168 * v5.9, in arch/powerpc/mm/numa.c, function __node_distance(). 169 */ 170 if (n_level == 0) { 171 continue; 172 } 173 174 /* 175 * We must assign all assoc_src to dst, starting from n_level 176 * and going up to 0x1. 177 */ 178 for (i = n_level; i > 0; i--) { 179 assoc_src = spapr->numa_assoc_array[src][i]; 180 spapr->numa_assoc_array[dst][i] = assoc_src; 181 } 182 } 183 } 184 185 } 186 187 /* 188 * Set NUMA machine state data based on FORM1 affinity semantics. 189 */ 190 static void spapr_numa_FORM1_affinity_init(SpaprMachineState *spapr, 191 MachineState *machine) 192 { 193 SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr); 194 int nb_numa_nodes = machine->numa_state->num_nodes; 195 int i, j, max_nodes_with_gpus; 196 197 /* 198 * For all associativity arrays: first position is the size, 199 * position FORM1_DIST_REF_POINTS is always the numa_id, 200 * represented by the index 'i'. 201 * 202 * This will break on sparse NUMA setups, when/if QEMU starts 203 * to support it, because there will be no more guarantee that 204 * 'i' will be a valid node_id set by the user. 205 */ 206 for (i = 0; i < nb_numa_nodes; i++) { 207 spapr->numa_assoc_array[i][0] = cpu_to_be32(FORM1_DIST_REF_POINTS); 208 spapr->numa_assoc_array[i][FORM1_DIST_REF_POINTS] = cpu_to_be32(i); 209 } 210 211 /* 212 * Initialize NVLink GPU associativity arrays. We know that 213 * the first GPU will take the first available NUMA id, and 214 * we'll have a maximum of NVGPU_MAX_NUM GPUs in the machine. 215 * At this point we're not sure if there are GPUs or not, but 216 * let's initialize the associativity arrays and allow NVLink 217 * GPUs to be handled like regular NUMA nodes later on. 218 */ 219 max_nodes_with_gpus = nb_numa_nodes + NVGPU_MAX_NUM; 220 221 for (i = nb_numa_nodes; i < max_nodes_with_gpus; i++) { 222 spapr->numa_assoc_array[i][0] = cpu_to_be32(FORM1_DIST_REF_POINTS); 223 224 for (j = 1; j < FORM1_DIST_REF_POINTS; j++) { 225 uint32_t gpu_assoc = smc->pre_5_1_assoc_refpoints ? 226 SPAPR_GPU_NUMA_ID : cpu_to_be32(i); 227 spapr->numa_assoc_array[i][j] = gpu_assoc; 228 } 229 230 spapr->numa_assoc_array[i][FORM1_DIST_REF_POINTS] = cpu_to_be32(i); 231 } 232 233 /* 234 * Guests pseries-5.1 and older uses zeroed associativity domains, 235 * i.e. no domain definition based on NUMA distance input. 236 * 237 * Same thing with guests that have only one NUMA node. 238 */ 239 if (smc->pre_5_2_numa_associativity || 240 machine->numa_state->num_nodes <= 1) { 241 return; 242 } 243 244 if (!spapr_numa_is_symmetrical(machine)) { 245 error_report("Asymmetrical NUMA topologies aren't supported " 246 "in the pSeries machine"); 247 exit(EXIT_FAILURE); 248 } 249 250 spapr_numa_define_FORM1_domains(spapr); 251 } 252 253 void spapr_numa_associativity_init(SpaprMachineState *spapr, 254 MachineState *machine) 255 { 256 spapr_numa_FORM1_affinity_init(spapr, machine); 257 } 258 259 void spapr_numa_write_associativity_dt(SpaprMachineState *spapr, void *fdt, 260 int offset, int nodeid) 261 { 262 _FDT((fdt_setprop(fdt, offset, "ibm,associativity", 263 spapr->numa_assoc_array[nodeid], 264 get_numa_assoc_size(spapr) * sizeof(uint32_t)))); 265 } 266 267 static uint32_t *spapr_numa_get_vcpu_assoc(SpaprMachineState *spapr, 268 PowerPCCPU *cpu) 269 { 270 int max_distance_ref_points = get_max_dist_ref_points(spapr); 271 int vcpu_assoc_size = get_vcpu_assoc_size(spapr); 272 uint32_t *vcpu_assoc = g_new(uint32_t, vcpu_assoc_size); 273 int index = spapr_get_vcpu_id(cpu); 274 275 /* 276 * VCPUs have an extra 'cpu_id' value in ibm,associativity 277 * compared to other resources. Increment the size at index 278 * 0, put cpu_id last, then copy the remaining associativity 279 * domains. 280 */ 281 vcpu_assoc[0] = cpu_to_be32(max_distance_ref_points + 1); 282 vcpu_assoc[vcpu_assoc_size - 1] = cpu_to_be32(index); 283 memcpy(vcpu_assoc + 1, spapr->numa_assoc_array[cpu->node_id] + 1, 284 (vcpu_assoc_size - 2) * sizeof(uint32_t)); 285 286 return vcpu_assoc; 287 } 288 289 int spapr_numa_fixup_cpu_dt(SpaprMachineState *spapr, void *fdt, 290 int offset, PowerPCCPU *cpu) 291 { 292 g_autofree uint32_t *vcpu_assoc = NULL; 293 int vcpu_assoc_size = get_vcpu_assoc_size(spapr); 294 295 vcpu_assoc = spapr_numa_get_vcpu_assoc(spapr, cpu); 296 297 /* Advertise NUMA via ibm,associativity */ 298 return fdt_setprop(fdt, offset, "ibm,associativity", vcpu_assoc, 299 vcpu_assoc_size * sizeof(uint32_t)); 300 } 301 302 303 int spapr_numa_write_assoc_lookup_arrays(SpaprMachineState *spapr, void *fdt, 304 int offset) 305 { 306 MachineState *machine = MACHINE(spapr); 307 int max_distance_ref_points = get_max_dist_ref_points(spapr); 308 int nb_numa_nodes = machine->numa_state->num_nodes; 309 int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1; 310 uint32_t *int_buf, *cur_index, buf_len; 311 int ret, i; 312 313 /* ibm,associativity-lookup-arrays */ 314 buf_len = (nr_nodes * max_distance_ref_points + 2) * sizeof(uint32_t); 315 cur_index = int_buf = g_malloc0(buf_len); 316 int_buf[0] = cpu_to_be32(nr_nodes); 317 /* Number of entries per associativity list */ 318 int_buf[1] = cpu_to_be32(max_distance_ref_points); 319 cur_index += 2; 320 for (i = 0; i < nr_nodes; i++) { 321 /* 322 * For the lookup-array we use the ibm,associativity array, 323 * from numa_assoc_array. without the first element (size). 324 */ 325 uint32_t *associativity = spapr->numa_assoc_array[i]; 326 memcpy(cur_index, ++associativity, 327 sizeof(uint32_t) * max_distance_ref_points); 328 cur_index += max_distance_ref_points; 329 } 330 ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf, 331 (cur_index - int_buf) * sizeof(uint32_t)); 332 g_free(int_buf); 333 334 return ret; 335 } 336 337 static void spapr_numa_FORM1_write_rtas_dt(SpaprMachineState *spapr, 338 void *fdt, int rtas) 339 { 340 MachineState *ms = MACHINE(spapr); 341 SpaprMachineClass *smc = SPAPR_MACHINE_GET_CLASS(spapr); 342 uint32_t number_nvgpus_nodes = spapr->gpu_numa_id - 343 spapr_numa_initial_nvgpu_numa_id(ms); 344 uint32_t refpoints[] = { 345 cpu_to_be32(0x4), 346 cpu_to_be32(0x3), 347 cpu_to_be32(0x2), 348 cpu_to_be32(0x1), 349 }; 350 uint32_t nr_refpoints = ARRAY_SIZE(refpoints); 351 uint32_t maxdomain = ms->numa_state->num_nodes + number_nvgpus_nodes; 352 uint32_t maxdomains[] = { 353 cpu_to_be32(4), 354 cpu_to_be32(maxdomain), 355 cpu_to_be32(maxdomain), 356 cpu_to_be32(maxdomain), 357 cpu_to_be32(maxdomain) 358 }; 359 360 if (smc->pre_5_2_numa_associativity || 361 ms->numa_state->num_nodes <= 1) { 362 uint32_t legacy_refpoints[] = { 363 cpu_to_be32(0x4), 364 cpu_to_be32(0x4), 365 cpu_to_be32(0x2), 366 }; 367 uint32_t legacy_maxdomain = spapr->gpu_numa_id > 1 ? 1 : 0; 368 uint32_t legacy_maxdomains[] = { 369 cpu_to_be32(4), 370 cpu_to_be32(legacy_maxdomain), 371 cpu_to_be32(legacy_maxdomain), 372 cpu_to_be32(legacy_maxdomain), 373 cpu_to_be32(spapr->gpu_numa_id), 374 }; 375 376 G_STATIC_ASSERT(sizeof(legacy_refpoints) <= sizeof(refpoints)); 377 G_STATIC_ASSERT(sizeof(legacy_maxdomains) <= sizeof(maxdomains)); 378 379 nr_refpoints = 3; 380 381 memcpy(refpoints, legacy_refpoints, sizeof(legacy_refpoints)); 382 memcpy(maxdomains, legacy_maxdomains, sizeof(legacy_maxdomains)); 383 384 /* pseries-5.0 and older reference-points array is {0x4, 0x4} */ 385 if (smc->pre_5_1_assoc_refpoints) { 386 nr_refpoints = 2; 387 } 388 } 389 390 _FDT(fdt_setprop(fdt, rtas, "ibm,associativity-reference-points", 391 refpoints, nr_refpoints * sizeof(refpoints[0]))); 392 393 _FDT(fdt_setprop(fdt, rtas, "ibm,max-associativity-domains", 394 maxdomains, sizeof(maxdomains))); 395 } 396 397 /* 398 * Helper that writes ibm,associativity-reference-points and 399 * max-associativity-domains in the RTAS pointed by @rtas 400 * in the DT @fdt. 401 */ 402 void spapr_numa_write_rtas_dt(SpaprMachineState *spapr, void *fdt, int rtas) 403 { 404 spapr_numa_FORM1_write_rtas_dt(spapr, fdt, rtas); 405 } 406 407 static target_ulong h_home_node_associativity(PowerPCCPU *cpu, 408 SpaprMachineState *spapr, 409 target_ulong opcode, 410 target_ulong *args) 411 { 412 g_autofree uint32_t *vcpu_assoc = NULL; 413 target_ulong flags = args[0]; 414 target_ulong procno = args[1]; 415 PowerPCCPU *tcpu; 416 int idx, assoc_idx; 417 int vcpu_assoc_size = get_vcpu_assoc_size(spapr); 418 419 /* only support procno from H_REGISTER_VPA */ 420 if (flags != 0x1) { 421 return H_FUNCTION; 422 } 423 424 tcpu = spapr_find_cpu(procno); 425 if (tcpu == NULL) { 426 return H_P2; 427 } 428 429 /* 430 * Given that we want to be flexible with the sizes and indexes, 431 * we must consider that there is a hard limit of how many 432 * associativities domain we can fit in R4 up to R9, which would be 433 * 12 associativity domains for vcpus. Assert and bail if that's 434 * not the case. 435 */ 436 g_assert((vcpu_assoc_size - 1) <= 12); 437 438 vcpu_assoc = spapr_numa_get_vcpu_assoc(spapr, tcpu); 439 /* assoc_idx starts at 1 to skip associativity size */ 440 assoc_idx = 1; 441 442 #define ASSOCIATIVITY(a, b) (((uint64_t)(a) << 32) | \ 443 ((uint64_t)(b) & 0xffffffff)) 444 445 for (idx = 0; idx < 6; idx++) { 446 int32_t a, b; 447 448 /* 449 * vcpu_assoc[] will contain the associativity domains for tcpu, 450 * including tcpu->node_id and procno, meaning that we don't 451 * need to use these variables here. 452 * 453 * We'll read 2 values at a time to fill up the ASSOCIATIVITY() 454 * macro. The ternary will fill the remaining registers with -1 455 * after we went through vcpu_assoc[]. 456 */ 457 a = assoc_idx < vcpu_assoc_size ? 458 be32_to_cpu(vcpu_assoc[assoc_idx++]) : -1; 459 b = assoc_idx < vcpu_assoc_size ? 460 be32_to_cpu(vcpu_assoc[assoc_idx++]) : -1; 461 462 args[idx] = ASSOCIATIVITY(a, b); 463 } 464 #undef ASSOCIATIVITY 465 466 return H_SUCCESS; 467 } 468 469 static void spapr_numa_register_types(void) 470 { 471 /* Virtual Processor Home Node */ 472 spapr_register_hypercall(H_HOME_NODE_ASSOCIATIVITY, 473 h_home_node_associativity); 474 } 475 476 type_init(spapr_numa_register_types) 477