1 /*
2 * CXL Type 3 (memory expander) device
3 *
4 * Copyright(C) 2020 Intel Corporation.
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2. See the
7 * COPYING file in the top-level directory.
8 *
9 * SPDX-License-Identifier: GPL-v2-only
10 */
11
12 #include "qemu/osdep.h"
13 #include "qemu/units.h"
14 #include "qemu/error-report.h"
15 #include "qapi/qapi-commands-cxl.h"
16 #include "hw/mem/memory-device.h"
17 #include "hw/mem/pc-dimm.h"
18 #include "hw/pci/pci.h"
19 #include "hw/qdev-properties.h"
20 #include "qapi/error.h"
21 #include "qemu/log.h"
22 #include "qemu/module.h"
23 #include "qemu/pmem.h"
24 #include "qemu/range.h"
25 #include "qemu/rcu.h"
26 #include "qemu/guest-random.h"
27 #include "sysemu/hostmem.h"
28 #include "sysemu/numa.h"
29 #include "hw/cxl/cxl.h"
30 #include "hw/pci/msix.h"
31
32 #define DWORD_BYTE 4
33 #define CXL_CAPACITY_MULTIPLIER (256 * MiB)
34
35 /* Default CDAT entries for a memory region */
36 enum {
37 CT3_CDAT_DSMAS,
38 CT3_CDAT_DSLBIS0,
39 CT3_CDAT_DSLBIS1,
40 CT3_CDAT_DSLBIS2,
41 CT3_CDAT_DSLBIS3,
42 CT3_CDAT_DSEMTS,
43 CT3_CDAT_NUM_ENTRIES
44 };
45
ct3_build_cdat_entries_for_mr(CDATSubHeader ** cdat_table,int dsmad_handle,uint64_t size,bool is_pmem,bool is_dynamic,uint64_t dpa_base)46 static void ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
47 int dsmad_handle, uint64_t size,
48 bool is_pmem, bool is_dynamic,
49 uint64_t dpa_base)
50 {
51 CDATDsmas *dsmas;
52 CDATDslbis *dslbis0;
53 CDATDslbis *dslbis1;
54 CDATDslbis *dslbis2;
55 CDATDslbis *dslbis3;
56 CDATDsemts *dsemts;
57
58 dsmas = g_malloc(sizeof(*dsmas));
59 *dsmas = (CDATDsmas) {
60 .header = {
61 .type = CDAT_TYPE_DSMAS,
62 .length = sizeof(*dsmas),
63 },
64 .DSMADhandle = dsmad_handle,
65 .flags = (is_pmem ? CDAT_DSMAS_FLAG_NV : 0) |
66 (is_dynamic ? CDAT_DSMAS_FLAG_DYNAMIC_CAP : 0),
67 .DPA_base = dpa_base,
68 .DPA_length = size,
69 };
70
71 /* For now, no memory side cache, plausiblish numbers */
72 dslbis0 = g_malloc(sizeof(*dslbis0));
73 *dslbis0 = (CDATDslbis) {
74 .header = {
75 .type = CDAT_TYPE_DSLBIS,
76 .length = sizeof(*dslbis0),
77 },
78 .handle = dsmad_handle,
79 .flags = HMAT_LB_MEM_MEMORY,
80 .data_type = HMAT_LB_DATA_READ_LATENCY,
81 .entry_base_unit = 10000, /* 10ns base */
82 .entry[0] = 15, /* 150ns */
83 };
84
85 dslbis1 = g_malloc(sizeof(*dslbis1));
86 *dslbis1 = (CDATDslbis) {
87 .header = {
88 .type = CDAT_TYPE_DSLBIS,
89 .length = sizeof(*dslbis1),
90 },
91 .handle = dsmad_handle,
92 .flags = HMAT_LB_MEM_MEMORY,
93 .data_type = HMAT_LB_DATA_WRITE_LATENCY,
94 .entry_base_unit = 10000,
95 .entry[0] = 25, /* 250ns */
96 };
97
98 dslbis2 = g_malloc(sizeof(*dslbis2));
99 *dslbis2 = (CDATDslbis) {
100 .header = {
101 .type = CDAT_TYPE_DSLBIS,
102 .length = sizeof(*dslbis2),
103 },
104 .handle = dsmad_handle,
105 .flags = HMAT_LB_MEM_MEMORY,
106 .data_type = HMAT_LB_DATA_READ_BANDWIDTH,
107 .entry_base_unit = 1000, /* GB/s */
108 .entry[0] = 16,
109 };
110
111 dslbis3 = g_malloc(sizeof(*dslbis3));
112 *dslbis3 = (CDATDslbis) {
113 .header = {
114 .type = CDAT_TYPE_DSLBIS,
115 .length = sizeof(*dslbis3),
116 },
117 .handle = dsmad_handle,
118 .flags = HMAT_LB_MEM_MEMORY,
119 .data_type = HMAT_LB_DATA_WRITE_BANDWIDTH,
120 .entry_base_unit = 1000, /* GB/s */
121 .entry[0] = 16,
122 };
123
124 dsemts = g_malloc(sizeof(*dsemts));
125 *dsemts = (CDATDsemts) {
126 .header = {
127 .type = CDAT_TYPE_DSEMTS,
128 .length = sizeof(*dsemts),
129 },
130 .DSMAS_handle = dsmad_handle,
131 /*
132 * NV: Reserved - the non volatile from DSMAS matters
133 * V: EFI_MEMORY_SP
134 */
135 .EFI_memory_type_attr = is_pmem ? 2 : 1,
136 .DPA_offset = 0,
137 .DPA_length = size,
138 };
139
140 /* Header always at start of structure */
141 cdat_table[CT3_CDAT_DSMAS] = (CDATSubHeader *)dsmas;
142 cdat_table[CT3_CDAT_DSLBIS0] = (CDATSubHeader *)dslbis0;
143 cdat_table[CT3_CDAT_DSLBIS1] = (CDATSubHeader *)dslbis1;
144 cdat_table[CT3_CDAT_DSLBIS2] = (CDATSubHeader *)dslbis2;
145 cdat_table[CT3_CDAT_DSLBIS3] = (CDATSubHeader *)dslbis3;
146 cdat_table[CT3_CDAT_DSEMTS] = (CDATSubHeader *)dsemts;
147 }
148
ct3_build_cdat_table(CDATSubHeader *** cdat_table,void * priv)149 static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
150 {
151 g_autofree CDATSubHeader **table = NULL;
152 CXLType3Dev *ct3d = priv;
153 MemoryRegion *volatile_mr = NULL, *nonvolatile_mr = NULL;
154 MemoryRegion *dc_mr = NULL;
155 uint64_t vmr_size = 0, pmr_size = 0;
156 int dsmad_handle = 0;
157 int cur_ent = 0;
158 int len = 0;
159
160 if (!ct3d->hostpmem && !ct3d->hostvmem && !ct3d->dc.num_regions) {
161 return 0;
162 }
163
164 if (ct3d->hostvmem) {
165 volatile_mr = host_memory_backend_get_memory(ct3d->hostvmem);
166 if (!volatile_mr) {
167 return -EINVAL;
168 }
169 len += CT3_CDAT_NUM_ENTRIES;
170 vmr_size = memory_region_size(volatile_mr);
171 }
172
173 if (ct3d->hostpmem) {
174 nonvolatile_mr = host_memory_backend_get_memory(ct3d->hostpmem);
175 if (!nonvolatile_mr) {
176 return -EINVAL;
177 }
178 len += CT3_CDAT_NUM_ENTRIES;
179 pmr_size = memory_region_size(nonvolatile_mr);
180 }
181
182 if (ct3d->dc.num_regions) {
183 if (!ct3d->dc.host_dc) {
184 return -EINVAL;
185 }
186 dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
187 if (!dc_mr) {
188 return -EINVAL;
189 }
190 len += CT3_CDAT_NUM_ENTRIES * ct3d->dc.num_regions;
191 }
192
193 table = g_malloc0(len * sizeof(*table));
194
195 /* Now fill them in */
196 if (volatile_mr) {
197 ct3_build_cdat_entries_for_mr(table, dsmad_handle++, vmr_size,
198 false, false, 0);
199 cur_ent = CT3_CDAT_NUM_ENTRIES;
200 }
201
202 if (nonvolatile_mr) {
203 uint64_t base = vmr_size;
204 ct3_build_cdat_entries_for_mr(&(table[cur_ent]), dsmad_handle++,
205 pmr_size, true, false, base);
206 cur_ent += CT3_CDAT_NUM_ENTRIES;
207 }
208
209 if (dc_mr) {
210 int i;
211 uint64_t region_base = vmr_size + pmr_size;
212
213 /*
214 * We assume the dynamic capacity to be volatile for now.
215 * Non-volatile dynamic capacity will be added if needed in the
216 * future.
217 */
218 for (i = 0; i < ct3d->dc.num_regions; i++) {
219 ct3_build_cdat_entries_for_mr(&(table[cur_ent]),
220 dsmad_handle++,
221 ct3d->dc.regions[i].len,
222 false, true, region_base);
223 ct3d->dc.regions[i].dsmadhandle = dsmad_handle - 1;
224
225 cur_ent += CT3_CDAT_NUM_ENTRIES;
226 region_base += ct3d->dc.regions[i].len;
227 }
228 }
229
230 assert(len == cur_ent);
231
232 *cdat_table = g_steal_pointer(&table);
233
234 return len;
235 }
236
ct3_free_cdat_table(CDATSubHeader ** cdat_table,int num,void * priv)237 static void ct3_free_cdat_table(CDATSubHeader **cdat_table, int num, void *priv)
238 {
239 int i;
240
241 for (i = 0; i < num; i++) {
242 g_free(cdat_table[i]);
243 }
244 g_free(cdat_table);
245 }
246
cxl_doe_cdat_rsp(DOECap * doe_cap)247 static bool cxl_doe_cdat_rsp(DOECap *doe_cap)
248 {
249 CDATObject *cdat = &CXL_TYPE3(doe_cap->pdev)->cxl_cstate.cdat;
250 uint16_t ent;
251 void *base;
252 uint32_t len;
253 CDATReq *req = pcie_doe_get_write_mbox_ptr(doe_cap);
254 CDATRsp rsp;
255
256 assert(cdat->entry_len);
257
258 /* Discard if request length mismatched */
259 if (pcie_doe_get_obj_len(req) <
260 DIV_ROUND_UP(sizeof(CDATReq), DWORD_BYTE)) {
261 return false;
262 }
263
264 ent = req->entry_handle;
265 base = cdat->entry[ent].base;
266 len = cdat->entry[ent].length;
267
268 rsp = (CDATRsp) {
269 .header = {
270 .vendor_id = CXL_VENDOR_ID,
271 .data_obj_type = CXL_DOE_TABLE_ACCESS,
272 .reserved = 0x0,
273 .length = DIV_ROUND_UP((sizeof(rsp) + len), DWORD_BYTE),
274 },
275 .rsp_code = CXL_DOE_TAB_RSP,
276 .table_type = CXL_DOE_TAB_TYPE_CDAT,
277 .entry_handle = (ent < cdat->entry_len - 1) ?
278 ent + 1 : CXL_DOE_TAB_ENT_MAX,
279 };
280
281 memcpy(doe_cap->read_mbox, &rsp, sizeof(rsp));
282 memcpy(doe_cap->read_mbox + DIV_ROUND_UP(sizeof(rsp), DWORD_BYTE),
283 base, len);
284
285 doe_cap->read_mbox_len += rsp.header.length;
286
287 return true;
288 }
289
ct3d_config_read(PCIDevice * pci_dev,uint32_t addr,int size)290 static uint32_t ct3d_config_read(PCIDevice *pci_dev, uint32_t addr, int size)
291 {
292 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev);
293 uint32_t val;
294
295 if (pcie_doe_read_config(&ct3d->doe_cdat, addr, size, &val)) {
296 return val;
297 }
298
299 return pci_default_read_config(pci_dev, addr, size);
300 }
301
ct3d_config_write(PCIDevice * pci_dev,uint32_t addr,uint32_t val,int size)302 static void ct3d_config_write(PCIDevice *pci_dev, uint32_t addr, uint32_t val,
303 int size)
304 {
305 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev);
306
307 pcie_doe_write_config(&ct3d->doe_cdat, addr, val, size);
308 pci_default_write_config(pci_dev, addr, val, size);
309 pcie_aer_write_config(pci_dev, addr, val, size);
310 }
311
312 /*
313 * Null value of all Fs suggested by IEEE RA guidelines for use of
314 * EU, OUI and CID
315 */
316 #define UI64_NULL ~(0ULL)
317
build_dvsecs(CXLType3Dev * ct3d)318 static void build_dvsecs(CXLType3Dev *ct3d)
319 {
320 CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;
321 uint8_t *dvsec;
322 uint32_t range1_size_hi, range1_size_lo,
323 range1_base_hi = 0, range1_base_lo = 0,
324 range2_size_hi = 0, range2_size_lo = 0,
325 range2_base_hi = 0, range2_base_lo = 0;
326
327 /*
328 * Volatile memory is mapped as (0x0)
329 * Persistent memory is mapped at (volatile->size)
330 */
331 if (ct3d->hostvmem) {
332 range1_size_hi = ct3d->hostvmem->size >> 32;
333 range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
334 (ct3d->hostvmem->size & 0xF0000000);
335 if (ct3d->hostpmem) {
336 range2_size_hi = ct3d->hostpmem->size >> 32;
337 range2_size_lo = (2 << 5) | (2 << 2) | 0x3 |
338 (ct3d->hostpmem->size & 0xF0000000);
339 }
340 } else if (ct3d->hostpmem) {
341 range1_size_hi = ct3d->hostpmem->size >> 32;
342 range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
343 (ct3d->hostpmem->size & 0xF0000000);
344 } else {
345 /*
346 * For DCD with no static memory, set memory active, memory class bits.
347 * No range is set.
348 */
349 range1_size_hi = 0;
350 range1_size_lo = (2 << 5) | (2 << 2) | 0x3;
351 }
352
353 dvsec = (uint8_t *)&(CXLDVSECDevice){
354 .cap = 0x1e,
355 .ctrl = 0x2,
356 .status2 = 0x2,
357 .range1_size_hi = range1_size_hi,
358 .range1_size_lo = range1_size_lo,
359 .range1_base_hi = range1_base_hi,
360 .range1_base_lo = range1_base_lo,
361 .range2_size_hi = range2_size_hi,
362 .range2_size_lo = range2_size_lo,
363 .range2_base_hi = range2_base_hi,
364 .range2_base_lo = range2_base_lo,
365 };
366 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE,
367 PCIE_CXL_DEVICE_DVSEC_LENGTH,
368 PCIE_CXL_DEVICE_DVSEC,
369 PCIE_CXL31_DEVICE_DVSEC_REVID, dvsec);
370
371 dvsec = (uint8_t *)&(CXLDVSECRegisterLocator){
372 .rsvd = 0,
373 .reg0_base_lo = RBI_COMPONENT_REG | CXL_COMPONENT_REG_BAR_IDX,
374 .reg0_base_hi = 0,
375 .reg1_base_lo = RBI_CXL_DEVICE_REG | CXL_DEVICE_REG_BAR_IDX,
376 .reg1_base_hi = 0,
377 };
378 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE,
379 REG_LOC_DVSEC_LENGTH, REG_LOC_DVSEC,
380 REG_LOC_DVSEC_REVID, dvsec);
381 dvsec = (uint8_t *)&(CXLDVSECDeviceGPF){
382 .phase2_duration = 0x603, /* 3 seconds */
383 .phase2_power = 0x33, /* 0x33 miliwatts */
384 };
385 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE,
386 GPF_DEVICE_DVSEC_LENGTH, GPF_DEVICE_DVSEC,
387 GPF_DEVICE_DVSEC_REVID, dvsec);
388
389 dvsec = (uint8_t *)&(CXLDVSECPortFlexBus){
390 .cap = 0x26, /* 68B, IO, Mem, non-MLD */
391 .ctrl = 0x02, /* IO always enabled */
392 .status = 0x26, /* same as capabilities */
393 .rcvd_mod_ts_data_phase1 = 0xef, /* WTF? */
394 };
395 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE,
396 PCIE_CXL3_FLEXBUS_PORT_DVSEC_LENGTH,
397 PCIE_FLEXBUS_PORT_DVSEC,
398 PCIE_CXL3_FLEXBUS_PORT_DVSEC_REVID, dvsec);
399 }
400
hdm_decoder_commit(CXLType3Dev * ct3d,int which)401 static void hdm_decoder_commit(CXLType3Dev *ct3d, int which)
402 {
403 int hdm_inc = R_CXL_HDM_DECODER1_BASE_LO - R_CXL_HDM_DECODER0_BASE_LO;
404 ComponentRegisters *cregs = &ct3d->cxl_cstate.crb;
405 uint32_t *cache_mem = cregs->cache_mem_registers;
406 uint32_t ctrl;
407
408 ctrl = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc);
409 /* TODO: Sanity checks that the decoder is possible */
410 ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, ERR, 0);
411 ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 1);
412
413 stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl);
414 }
415
hdm_decoder_uncommit(CXLType3Dev * ct3d,int which)416 static void hdm_decoder_uncommit(CXLType3Dev *ct3d, int which)
417 {
418 int hdm_inc = R_CXL_HDM_DECODER1_BASE_LO - R_CXL_HDM_DECODER0_BASE_LO;
419 ComponentRegisters *cregs = &ct3d->cxl_cstate.crb;
420 uint32_t *cache_mem = cregs->cache_mem_registers;
421 uint32_t ctrl;
422
423 ctrl = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc);
424
425 ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, ERR, 0);
426 ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 0);
427
428 stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl);
429 }
430
ct3d_qmp_uncor_err_to_cxl(CxlUncorErrorType qmp_err)431 static int ct3d_qmp_uncor_err_to_cxl(CxlUncorErrorType qmp_err)
432 {
433 switch (qmp_err) {
434 case CXL_UNCOR_ERROR_TYPE_CACHE_DATA_PARITY:
435 return CXL_RAS_UNC_ERR_CACHE_DATA_PARITY;
436 case CXL_UNCOR_ERROR_TYPE_CACHE_ADDRESS_PARITY:
437 return CXL_RAS_UNC_ERR_CACHE_ADDRESS_PARITY;
438 case CXL_UNCOR_ERROR_TYPE_CACHE_BE_PARITY:
439 return CXL_RAS_UNC_ERR_CACHE_BE_PARITY;
440 case CXL_UNCOR_ERROR_TYPE_CACHE_DATA_ECC:
441 return CXL_RAS_UNC_ERR_CACHE_DATA_ECC;
442 case CXL_UNCOR_ERROR_TYPE_MEM_DATA_PARITY:
443 return CXL_RAS_UNC_ERR_MEM_DATA_PARITY;
444 case CXL_UNCOR_ERROR_TYPE_MEM_ADDRESS_PARITY:
445 return CXL_RAS_UNC_ERR_MEM_ADDRESS_PARITY;
446 case CXL_UNCOR_ERROR_TYPE_MEM_BE_PARITY:
447 return CXL_RAS_UNC_ERR_MEM_BE_PARITY;
448 case CXL_UNCOR_ERROR_TYPE_MEM_DATA_ECC:
449 return CXL_RAS_UNC_ERR_MEM_DATA_ECC;
450 case CXL_UNCOR_ERROR_TYPE_REINIT_THRESHOLD:
451 return CXL_RAS_UNC_ERR_REINIT_THRESHOLD;
452 case CXL_UNCOR_ERROR_TYPE_RSVD_ENCODING:
453 return CXL_RAS_UNC_ERR_RSVD_ENCODING;
454 case CXL_UNCOR_ERROR_TYPE_POISON_RECEIVED:
455 return CXL_RAS_UNC_ERR_POISON_RECEIVED;
456 case CXL_UNCOR_ERROR_TYPE_RECEIVER_OVERFLOW:
457 return CXL_RAS_UNC_ERR_RECEIVER_OVERFLOW;
458 case CXL_UNCOR_ERROR_TYPE_INTERNAL:
459 return CXL_RAS_UNC_ERR_INTERNAL;
460 case CXL_UNCOR_ERROR_TYPE_CXL_IDE_TX:
461 return CXL_RAS_UNC_ERR_CXL_IDE_TX;
462 case CXL_UNCOR_ERROR_TYPE_CXL_IDE_RX:
463 return CXL_RAS_UNC_ERR_CXL_IDE_RX;
464 default:
465 return -EINVAL;
466 }
467 }
468
ct3d_qmp_cor_err_to_cxl(CxlCorErrorType qmp_err)469 static int ct3d_qmp_cor_err_to_cxl(CxlCorErrorType qmp_err)
470 {
471 switch (qmp_err) {
472 case CXL_COR_ERROR_TYPE_CACHE_DATA_ECC:
473 return CXL_RAS_COR_ERR_CACHE_DATA_ECC;
474 case CXL_COR_ERROR_TYPE_MEM_DATA_ECC:
475 return CXL_RAS_COR_ERR_MEM_DATA_ECC;
476 case CXL_COR_ERROR_TYPE_CRC_THRESHOLD:
477 return CXL_RAS_COR_ERR_CRC_THRESHOLD;
478 case CXL_COR_ERROR_TYPE_RETRY_THRESHOLD:
479 return CXL_RAS_COR_ERR_RETRY_THRESHOLD;
480 case CXL_COR_ERROR_TYPE_CACHE_POISON_RECEIVED:
481 return CXL_RAS_COR_ERR_CACHE_POISON_RECEIVED;
482 case CXL_COR_ERROR_TYPE_MEM_POISON_RECEIVED:
483 return CXL_RAS_COR_ERR_MEM_POISON_RECEIVED;
484 case CXL_COR_ERROR_TYPE_PHYSICAL:
485 return CXL_RAS_COR_ERR_PHYSICAL;
486 default:
487 return -EINVAL;
488 }
489 }
490
ct3d_reg_write(void * opaque,hwaddr offset,uint64_t value,unsigned size)491 static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value,
492 unsigned size)
493 {
494 CXLComponentState *cxl_cstate = opaque;
495 ComponentRegisters *cregs = &cxl_cstate->crb;
496 CXLType3Dev *ct3d = container_of(cxl_cstate, CXLType3Dev, cxl_cstate);
497 uint32_t *cache_mem = cregs->cache_mem_registers;
498 bool should_commit = false;
499 bool should_uncommit = false;
500 int which_hdm = -1;
501
502 assert(size == 4);
503 g_assert(offset < CXL2_COMPONENT_CM_REGION_SIZE);
504
505 switch (offset) {
506 case A_CXL_HDM_DECODER0_CTRL:
507 should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT);
508 should_uncommit = !should_commit;
509 which_hdm = 0;
510 break;
511 case A_CXL_HDM_DECODER1_CTRL:
512 should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT);
513 should_uncommit = !should_commit;
514 which_hdm = 1;
515 break;
516 case A_CXL_HDM_DECODER2_CTRL:
517 should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT);
518 should_uncommit = !should_commit;
519 which_hdm = 2;
520 break;
521 case A_CXL_HDM_DECODER3_CTRL:
522 should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT);
523 should_uncommit = !should_commit;
524 which_hdm = 3;
525 break;
526 case A_CXL_RAS_UNC_ERR_STATUS:
527 {
528 uint32_t capctrl = ldl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL);
529 uint32_t fe = FIELD_EX32(capctrl, CXL_RAS_ERR_CAP_CTRL,
530 FIRST_ERROR_POINTER);
531 CXLError *cxl_err;
532 uint32_t unc_err;
533
534 /*
535 * If single bit written that corresponds to the first error
536 * pointer being cleared, update the status and header log.
537 */
538 if (!QTAILQ_EMPTY(&ct3d->error_list)) {
539 if ((1 << fe) ^ value) {
540 CXLError *cxl_next;
541 /*
542 * Software is using wrong flow for multiple header recording
543 * Following behavior in PCIe r6.0 and assuming multiple
544 * header support. Implementation defined choice to clear all
545 * matching records if more than one bit set - which corresponds
546 * closest to behavior of hardware not capable of multiple
547 * header recording.
548 */
549 QTAILQ_FOREACH_SAFE(cxl_err, &ct3d->error_list, node,
550 cxl_next) {
551 if ((1 << cxl_err->type) & value) {
552 QTAILQ_REMOVE(&ct3d->error_list, cxl_err, node);
553 g_free(cxl_err);
554 }
555 }
556 } else {
557 /* Done with previous FE, so drop from list */
558 cxl_err = QTAILQ_FIRST(&ct3d->error_list);
559 QTAILQ_REMOVE(&ct3d->error_list, cxl_err, node);
560 g_free(cxl_err);
561 }
562
563 /*
564 * If there is another FE, then put that in place and update
565 * the header log
566 */
567 if (!QTAILQ_EMPTY(&ct3d->error_list)) {
568 uint32_t *header_log = &cache_mem[R_CXL_RAS_ERR_HEADER0];
569 int i;
570
571 cxl_err = QTAILQ_FIRST(&ct3d->error_list);
572 for (i = 0; i < CXL_RAS_ERR_HEADER_NUM; i++) {
573 stl_le_p(header_log + i, cxl_err->header[i]);
574 }
575 capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL,
576 FIRST_ERROR_POINTER, cxl_err->type);
577 } else {
578 /*
579 * If no more errors, then follow recommendation of PCI spec
580 * r6.0 6.2.4.2 to set the first error pointer to a status
581 * bit that will never be used.
582 */
583 capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL,
584 FIRST_ERROR_POINTER,
585 CXL_RAS_UNC_ERR_CXL_UNUSED);
586 }
587 stl_le_p((uint8_t *)cache_mem + A_CXL_RAS_ERR_CAP_CTRL, capctrl);
588 }
589 unc_err = 0;
590 QTAILQ_FOREACH(cxl_err, &ct3d->error_list, node) {
591 unc_err |= 1 << cxl_err->type;
592 }
593 stl_le_p((uint8_t *)cache_mem + offset, unc_err);
594
595 return;
596 }
597 case A_CXL_RAS_COR_ERR_STATUS:
598 {
599 uint32_t rw1c = value;
600 uint32_t temp = ldl_le_p((uint8_t *)cache_mem + offset);
601 temp &= ~rw1c;
602 stl_le_p((uint8_t *)cache_mem + offset, temp);
603 return;
604 }
605 default:
606 break;
607 }
608
609 stl_le_p((uint8_t *)cache_mem + offset, value);
610 if (should_commit) {
611 hdm_decoder_commit(ct3d, which_hdm);
612 } else if (should_uncommit) {
613 hdm_decoder_uncommit(ct3d, which_hdm);
614 }
615 }
616
617 /*
618 * TODO: dc region configuration will be updated once host backend and address
619 * space support is added for DCD.
620 */
cxl_create_dc_regions(CXLType3Dev * ct3d,Error ** errp)621 static bool cxl_create_dc_regions(CXLType3Dev *ct3d, Error **errp)
622 {
623 int i;
624 uint64_t region_base = 0;
625 uint64_t region_len;
626 uint64_t decode_len;
627 uint64_t blk_size = 2 * MiB;
628 CXLDCRegion *region;
629 MemoryRegion *mr;
630 uint64_t dc_size;
631
632 mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
633 dc_size = memory_region_size(mr);
634 region_len = DIV_ROUND_UP(dc_size, ct3d->dc.num_regions);
635
636 if (dc_size % (ct3d->dc.num_regions * CXL_CAPACITY_MULTIPLIER) != 0) {
637 error_setg(errp,
638 "backend size is not multiple of region len: 0x%" PRIx64,
639 region_len);
640 return false;
641 }
642 if (region_len % CXL_CAPACITY_MULTIPLIER != 0) {
643 error_setg(errp, "DC region size is unaligned to 0x%" PRIx64,
644 CXL_CAPACITY_MULTIPLIER);
645 return false;
646 }
647 decode_len = region_len;
648
649 if (ct3d->hostvmem) {
650 mr = host_memory_backend_get_memory(ct3d->hostvmem);
651 region_base += memory_region_size(mr);
652 }
653 if (ct3d->hostpmem) {
654 mr = host_memory_backend_get_memory(ct3d->hostpmem);
655 region_base += memory_region_size(mr);
656 }
657 if (region_base % CXL_CAPACITY_MULTIPLIER != 0) {
658 error_setg(errp, "DC region base not aligned to 0x%" PRIx64,
659 CXL_CAPACITY_MULTIPLIER);
660 return false;
661 }
662
663 for (i = 0, region = &ct3d->dc.regions[0];
664 i < ct3d->dc.num_regions;
665 i++, region++, region_base += region_len) {
666 *region = (CXLDCRegion) {
667 .base = region_base,
668 .decode_len = decode_len,
669 .len = region_len,
670 .block_size = blk_size,
671 /* dsmad_handle set when creating CDAT table entries */
672 .flags = 0,
673 };
674 ct3d->dc.total_capacity += region->len;
675 region->blk_bitmap = bitmap_new(region->len / region->block_size);
676 }
677 QTAILQ_INIT(&ct3d->dc.extents);
678 QTAILQ_INIT(&ct3d->dc.extents_pending);
679
680 return true;
681 }
682
cxl_destroy_dc_regions(CXLType3Dev * ct3d)683 static void cxl_destroy_dc_regions(CXLType3Dev *ct3d)
684 {
685 CXLDCExtent *ent, *ent_next;
686 CXLDCExtentGroup *group, *group_next;
687 int i;
688 CXLDCRegion *region;
689
690 QTAILQ_FOREACH_SAFE(ent, &ct3d->dc.extents, node, ent_next) {
691 cxl_remove_extent_from_extent_list(&ct3d->dc.extents, ent);
692 }
693
694 QTAILQ_FOREACH_SAFE(group, &ct3d->dc.extents_pending, node, group_next) {
695 QTAILQ_REMOVE(&ct3d->dc.extents_pending, group, node);
696 QTAILQ_FOREACH_SAFE(ent, &group->list, node, ent_next) {
697 cxl_remove_extent_from_extent_list(&group->list, ent);
698 }
699 g_free(group);
700 }
701
702 for (i = 0; i < ct3d->dc.num_regions; i++) {
703 region = &ct3d->dc.regions[i];
704 g_free(region->blk_bitmap);
705 }
706 }
707
cxl_setup_memory(CXLType3Dev * ct3d,Error ** errp)708 static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
709 {
710 DeviceState *ds = DEVICE(ct3d);
711
712 if (!ct3d->hostmem && !ct3d->hostvmem && !ct3d->hostpmem
713 && !ct3d->dc.num_regions) {
714 error_setg(errp, "at least one memdev property must be set");
715 return false;
716 } else if (ct3d->hostmem && ct3d->hostpmem) {
717 error_setg(errp, "[memdev] cannot be used with new "
718 "[persistent-memdev] property");
719 return false;
720 } else if (ct3d->hostmem) {
721 /* Use of hostmem property implies pmem */
722 ct3d->hostpmem = ct3d->hostmem;
723 ct3d->hostmem = NULL;
724 }
725
726 if (ct3d->hostpmem && !ct3d->lsa) {
727 error_setg(errp, "lsa property must be set for persistent devices");
728 return false;
729 }
730
731 if (ct3d->hostvmem) {
732 MemoryRegion *vmr;
733 char *v_name;
734
735 vmr = host_memory_backend_get_memory(ct3d->hostvmem);
736 if (!vmr) {
737 error_setg(errp, "volatile memdev must have backing device");
738 return false;
739 }
740 if (host_memory_backend_is_mapped(ct3d->hostvmem)) {
741 error_setg(errp, "memory backend %s can't be used multiple times.",
742 object_get_canonical_path_component(OBJECT(ct3d->hostvmem)));
743 return false;
744 }
745 memory_region_set_nonvolatile(vmr, false);
746 memory_region_set_enabled(vmr, true);
747 host_memory_backend_set_mapped(ct3d->hostvmem, true);
748 if (ds->id) {
749 v_name = g_strdup_printf("cxl-type3-dpa-vmem-space:%s", ds->id);
750 } else {
751 v_name = g_strdup("cxl-type3-dpa-vmem-space");
752 }
753 address_space_init(&ct3d->hostvmem_as, vmr, v_name);
754 ct3d->cxl_dstate.vmem_size = memory_region_size(vmr);
755 ct3d->cxl_dstate.static_mem_size += memory_region_size(vmr);
756 g_free(v_name);
757 }
758
759 if (ct3d->hostpmem) {
760 MemoryRegion *pmr;
761 char *p_name;
762
763 pmr = host_memory_backend_get_memory(ct3d->hostpmem);
764 if (!pmr) {
765 error_setg(errp, "persistent memdev must have backing device");
766 return false;
767 }
768 if (host_memory_backend_is_mapped(ct3d->hostpmem)) {
769 error_setg(errp, "memory backend %s can't be used multiple times.",
770 object_get_canonical_path_component(OBJECT(ct3d->hostpmem)));
771 return false;
772 }
773 memory_region_set_nonvolatile(pmr, true);
774 memory_region_set_enabled(pmr, true);
775 host_memory_backend_set_mapped(ct3d->hostpmem, true);
776 if (ds->id) {
777 p_name = g_strdup_printf("cxl-type3-dpa-pmem-space:%s", ds->id);
778 } else {
779 p_name = g_strdup("cxl-type3-dpa-pmem-space");
780 }
781 address_space_init(&ct3d->hostpmem_as, pmr, p_name);
782 ct3d->cxl_dstate.pmem_size = memory_region_size(pmr);
783 ct3d->cxl_dstate.static_mem_size += memory_region_size(pmr);
784 g_free(p_name);
785 }
786
787 ct3d->dc.total_capacity = 0;
788 if (ct3d->dc.num_regions > 0) {
789 MemoryRegion *dc_mr;
790 char *dc_name;
791
792 if (!ct3d->dc.host_dc) {
793 error_setg(errp, "dynamic capacity must have a backing device");
794 return false;
795 }
796
797 dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
798 if (!dc_mr) {
799 error_setg(errp, "dynamic capacity must have a backing device");
800 return false;
801 }
802
803 if (host_memory_backend_is_mapped(ct3d->dc.host_dc)) {
804 error_setg(errp, "memory backend %s can't be used multiple times.",
805 object_get_canonical_path_component(OBJECT(ct3d->dc.host_dc)));
806 return false;
807 }
808 /*
809 * Set DC regions as volatile for now, non-volatile support can
810 * be added in the future if needed.
811 */
812 memory_region_set_nonvolatile(dc_mr, false);
813 memory_region_set_enabled(dc_mr, true);
814 host_memory_backend_set_mapped(ct3d->dc.host_dc, true);
815 if (ds->id) {
816 dc_name = g_strdup_printf("cxl-dcd-dpa-dc-space:%s", ds->id);
817 } else {
818 dc_name = g_strdup("cxl-dcd-dpa-dc-space");
819 }
820 address_space_init(&ct3d->dc.host_dc_as, dc_mr, dc_name);
821 g_free(dc_name);
822
823 if (!cxl_create_dc_regions(ct3d, errp)) {
824 error_append_hint(errp, "setup DC regions failed");
825 return false;
826 }
827 }
828
829 return true;
830 }
831
832 static DOEProtocol doe_cdat_prot[] = {
833 { CXL_VENDOR_ID, CXL_DOE_TABLE_ACCESS, cxl_doe_cdat_rsp },
834 { }
835 };
836
ct3_realize(PCIDevice * pci_dev,Error ** errp)837 static void ct3_realize(PCIDevice *pci_dev, Error **errp)
838 {
839 ERRP_GUARD();
840 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev);
841 CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;
842 ComponentRegisters *regs = &cxl_cstate->crb;
843 MemoryRegion *mr = ®s->component_registers;
844 uint8_t *pci_conf = pci_dev->config;
845 unsigned short msix_num = 6;
846 int i, rc;
847 uint16_t count;
848
849 QTAILQ_INIT(&ct3d->error_list);
850
851 if (!cxl_setup_memory(ct3d, errp)) {
852 return;
853 }
854
855 pci_config_set_prog_interface(pci_conf, 0x10);
856
857 pcie_endpoint_cap_init(pci_dev, 0x80);
858 if (ct3d->sn != UI64_NULL) {
859 pcie_dev_ser_num_init(pci_dev, 0x100, ct3d->sn);
860 cxl_cstate->dvsec_offset = 0x100 + 0x0c;
861 } else {
862 cxl_cstate->dvsec_offset = 0x100;
863 }
864
865 ct3d->cxl_cstate.pdev = pci_dev;
866 build_dvsecs(ct3d);
867
868 regs->special_ops = g_new0(MemoryRegionOps, 1);
869 regs->special_ops->write = ct3d_reg_write;
870
871 cxl_component_register_block_init(OBJECT(pci_dev), cxl_cstate,
872 TYPE_CXL_TYPE3);
873
874 pci_register_bar(
875 pci_dev, CXL_COMPONENT_REG_BAR_IDX,
876 PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64, mr);
877
878 cxl_device_register_block_init(OBJECT(pci_dev), &ct3d->cxl_dstate,
879 &ct3d->cci);
880 pci_register_bar(pci_dev, CXL_DEVICE_REG_BAR_IDX,
881 PCI_BASE_ADDRESS_SPACE_MEMORY |
882 PCI_BASE_ADDRESS_MEM_TYPE_64,
883 &ct3d->cxl_dstate.device_registers);
884
885 /* MSI(-X) Initialization */
886 rc = msix_init_exclusive_bar(pci_dev, msix_num, 4, NULL);
887 if (rc) {
888 goto err_address_space_free;
889 }
890 for (i = 0; i < msix_num; i++) {
891 msix_vector_use(pci_dev, i);
892 }
893
894 /* DOE Initialization */
895 pcie_doe_init(pci_dev, &ct3d->doe_cdat, 0x190, doe_cdat_prot, true, 0);
896
897 cxl_cstate->cdat.build_cdat_table = ct3_build_cdat_table;
898 cxl_cstate->cdat.free_cdat_table = ct3_free_cdat_table;
899 cxl_cstate->cdat.private = ct3d;
900 if (!cxl_doe_cdat_init(cxl_cstate, errp)) {
901 goto err_free_special_ops;
902 }
903
904 pcie_cap_deverr_init(pci_dev);
905 /* Leave a bit of room for expansion */
906 rc = pcie_aer_init(pci_dev, PCI_ERR_VER, 0x200, PCI_ERR_SIZEOF, NULL);
907 if (rc) {
908 goto err_release_cdat;
909 }
910 cxl_event_init(&ct3d->cxl_dstate, 2);
911
912 /* Set default value for patrol scrub attributes */
913 ct3d->patrol_scrub_attrs.scrub_cycle_cap =
914 CXL_MEMDEV_PS_SCRUB_CYCLE_CHANGE_CAP_DEFAULT |
915 CXL_MEMDEV_PS_SCRUB_REALTIME_REPORT_CAP_DEFAULT;
916 ct3d->patrol_scrub_attrs.scrub_cycle =
917 CXL_MEMDEV_PS_CUR_SCRUB_CYCLE_DEFAULT |
918 (CXL_MEMDEV_PS_MIN_SCRUB_CYCLE_DEFAULT << 8);
919 ct3d->patrol_scrub_attrs.scrub_flags = CXL_MEMDEV_PS_ENABLE_DEFAULT;
920
921 /* Set default value for DDR5 ECS read attributes */
922 for (count = 0; count < CXL_ECS_NUM_MEDIA_FRUS; count++) {
923 ct3d->ecs_attrs[count].ecs_log_cap =
924 CXL_ECS_LOG_ENTRY_TYPE_DEFAULT;
925 ct3d->ecs_attrs[count].ecs_cap =
926 CXL_ECS_REALTIME_REPORT_CAP_DEFAULT;
927 ct3d->ecs_attrs[count].ecs_config =
928 CXL_ECS_THRESHOLD_COUNT_DEFAULT |
929 (CXL_ECS_MODE_DEFAULT << 3);
930 /* Reserved */
931 ct3d->ecs_attrs[count].ecs_flags = 0;
932 }
933
934 return;
935
936 err_release_cdat:
937 cxl_doe_cdat_release(cxl_cstate);
938 err_free_special_ops:
939 g_free(regs->special_ops);
940 err_address_space_free:
941 if (ct3d->dc.host_dc) {
942 cxl_destroy_dc_regions(ct3d);
943 address_space_destroy(&ct3d->dc.host_dc_as);
944 }
945 if (ct3d->hostpmem) {
946 address_space_destroy(&ct3d->hostpmem_as);
947 }
948 if (ct3d->hostvmem) {
949 address_space_destroy(&ct3d->hostvmem_as);
950 }
951 return;
952 }
953
ct3_exit(PCIDevice * pci_dev)954 static void ct3_exit(PCIDevice *pci_dev)
955 {
956 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev);
957 CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;
958 ComponentRegisters *regs = &cxl_cstate->crb;
959
960 pcie_aer_exit(pci_dev);
961 cxl_doe_cdat_release(cxl_cstate);
962 g_free(regs->special_ops);
963 if (ct3d->dc.host_dc) {
964 cxl_destroy_dc_regions(ct3d);
965 address_space_destroy(&ct3d->dc.host_dc_as);
966 }
967 if (ct3d->hostpmem) {
968 address_space_destroy(&ct3d->hostpmem_as);
969 }
970 if (ct3d->hostvmem) {
971 address_space_destroy(&ct3d->hostvmem_as);
972 }
973 }
974
975 /*
976 * Mark the DPA range [dpa, dap + len - 1] to be backed and accessible. This
977 * happens when a DC extent is added and accepted by the host.
978 */
ct3_set_region_block_backed(CXLType3Dev * ct3d,uint64_t dpa,uint64_t len)979 void ct3_set_region_block_backed(CXLType3Dev *ct3d, uint64_t dpa,
980 uint64_t len)
981 {
982 CXLDCRegion *region;
983
984 region = cxl_find_dc_region(ct3d, dpa, len);
985 if (!region) {
986 return;
987 }
988
989 bitmap_set(region->blk_bitmap, (dpa - region->base) / region->block_size,
990 len / region->block_size);
991 }
992
993 /*
994 * Check whether the DPA range [dpa, dpa + len - 1] is backed with DC extents.
995 * Used when validating read/write to dc regions
996 */
ct3_test_region_block_backed(CXLType3Dev * ct3d,uint64_t dpa,uint64_t len)997 bool ct3_test_region_block_backed(CXLType3Dev *ct3d, uint64_t dpa,
998 uint64_t len)
999 {
1000 CXLDCRegion *region;
1001 uint64_t nbits;
1002 long nr;
1003
1004 region = cxl_find_dc_region(ct3d, dpa, len);
1005 if (!region) {
1006 return false;
1007 }
1008
1009 nr = (dpa - region->base) / region->block_size;
1010 nbits = DIV_ROUND_UP(len, region->block_size);
1011 /*
1012 * if bits between [dpa, dpa + len) are all 1s, meaning the DPA range is
1013 * backed with DC extents, return true; else return false.
1014 */
1015 return find_next_zero_bit(region->blk_bitmap, nr + nbits, nr) == nr + nbits;
1016 }
1017
1018 /*
1019 * Mark the DPA range [dpa, dap + len - 1] to be unbacked and inaccessible.
1020 * This happens when a dc extent is released by the host.
1021 */
ct3_clear_region_block_backed(CXLType3Dev * ct3d,uint64_t dpa,uint64_t len)1022 void ct3_clear_region_block_backed(CXLType3Dev *ct3d, uint64_t dpa,
1023 uint64_t len)
1024 {
1025 CXLDCRegion *region;
1026 uint64_t nbits;
1027 long nr;
1028
1029 region = cxl_find_dc_region(ct3d, dpa, len);
1030 if (!region) {
1031 return;
1032 }
1033
1034 nr = (dpa - region->base) / region->block_size;
1035 nbits = len / region->block_size;
1036 bitmap_clear(region->blk_bitmap, nr, nbits);
1037 }
1038
cxl_type3_dpa(CXLType3Dev * ct3d,hwaddr host_addr,uint64_t * dpa)1039 static bool cxl_type3_dpa(CXLType3Dev *ct3d, hwaddr host_addr, uint64_t *dpa)
1040 {
1041 int hdm_inc = R_CXL_HDM_DECODER1_BASE_LO - R_CXL_HDM_DECODER0_BASE_LO;
1042 uint32_t *cache_mem = ct3d->cxl_cstate.crb.cache_mem_registers;
1043 unsigned int hdm_count;
1044 uint32_t cap;
1045 uint64_t dpa_base = 0;
1046 int i;
1047
1048 cap = ldl_le_p(cache_mem + R_CXL_HDM_DECODER_CAPABILITY);
1049 hdm_count = cxl_decoder_count_dec(FIELD_EX32(cap,
1050 CXL_HDM_DECODER_CAPABILITY,
1051 DECODER_COUNT));
1052
1053 for (i = 0; i < hdm_count; i++) {
1054 uint64_t decoder_base, decoder_size, hpa_offset, skip;
1055 uint32_t hdm_ctrl, low, high;
1056 int ig, iw;
1057
1058 low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_LO + i * hdm_inc);
1059 high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_HI + i * hdm_inc);
1060 decoder_base = ((uint64_t)high << 32) | (low & 0xf0000000);
1061
1062 low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_LO + i * hdm_inc);
1063 high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_HI + i * hdm_inc);
1064 decoder_size = ((uint64_t)high << 32) | (low & 0xf0000000);
1065
1066 low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_DPA_SKIP_LO +
1067 i * hdm_inc);
1068 high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_DPA_SKIP_HI +
1069 i * hdm_inc);
1070 skip = ((uint64_t)high << 32) | (low & 0xf0000000);
1071 dpa_base += skip;
1072
1073 hpa_offset = (uint64_t)host_addr - decoder_base;
1074
1075 hdm_ctrl = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + i * hdm_inc);
1076 iw = FIELD_EX32(hdm_ctrl, CXL_HDM_DECODER0_CTRL, IW);
1077 ig = FIELD_EX32(hdm_ctrl, CXL_HDM_DECODER0_CTRL, IG);
1078 if (!FIELD_EX32(hdm_ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED)) {
1079 return false;
1080 }
1081 if (((uint64_t)host_addr < decoder_base) ||
1082 (hpa_offset >= decoder_size)) {
1083 int decoded_iw = cxl_interleave_ways_dec(iw, &error_fatal);
1084
1085 if (decoded_iw == 0) {
1086 return false;
1087 }
1088
1089 dpa_base += decoder_size / decoded_iw;
1090 continue;
1091 }
1092
1093 *dpa = dpa_base +
1094 ((MAKE_64BIT_MASK(0, 8 + ig) & hpa_offset) |
1095 ((MAKE_64BIT_MASK(8 + ig + iw, 64 - 8 - ig - iw) & hpa_offset)
1096 >> iw));
1097
1098 return true;
1099 }
1100 return false;
1101 }
1102
cxl_type3_hpa_to_as_and_dpa(CXLType3Dev * ct3d,hwaddr host_addr,unsigned int size,AddressSpace ** as,uint64_t * dpa_offset)1103 static int cxl_type3_hpa_to_as_and_dpa(CXLType3Dev *ct3d,
1104 hwaddr host_addr,
1105 unsigned int size,
1106 AddressSpace **as,
1107 uint64_t *dpa_offset)
1108 {
1109 MemoryRegion *vmr = NULL, *pmr = NULL, *dc_mr = NULL;
1110 uint64_t vmr_size = 0, pmr_size = 0, dc_size = 0;
1111
1112 if (ct3d->hostvmem) {
1113 vmr = host_memory_backend_get_memory(ct3d->hostvmem);
1114 vmr_size = memory_region_size(vmr);
1115 }
1116 if (ct3d->hostpmem) {
1117 pmr = host_memory_backend_get_memory(ct3d->hostpmem);
1118 pmr_size = memory_region_size(pmr);
1119 }
1120 if (ct3d->dc.host_dc) {
1121 dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
1122 dc_size = memory_region_size(dc_mr);
1123 }
1124
1125 if (!vmr && !pmr && !dc_mr) {
1126 return -ENODEV;
1127 }
1128
1129 if (!cxl_type3_dpa(ct3d, host_addr, dpa_offset)) {
1130 return -EINVAL;
1131 }
1132
1133 if (*dpa_offset >= vmr_size + pmr_size + dc_size) {
1134 return -EINVAL;
1135 }
1136
1137 if (*dpa_offset < vmr_size) {
1138 *as = &ct3d->hostvmem_as;
1139 } else if (*dpa_offset < vmr_size + pmr_size) {
1140 *as = &ct3d->hostpmem_as;
1141 *dpa_offset -= vmr_size;
1142 } else {
1143 if (!ct3_test_region_block_backed(ct3d, *dpa_offset, size)) {
1144 return -ENODEV;
1145 }
1146
1147 *as = &ct3d->dc.host_dc_as;
1148 *dpa_offset -= (vmr_size + pmr_size);
1149 }
1150
1151 return 0;
1152 }
1153
cxl_type3_read(PCIDevice * d,hwaddr host_addr,uint64_t * data,unsigned size,MemTxAttrs attrs)1154 MemTxResult cxl_type3_read(PCIDevice *d, hwaddr host_addr, uint64_t *data,
1155 unsigned size, MemTxAttrs attrs)
1156 {
1157 CXLType3Dev *ct3d = CXL_TYPE3(d);
1158 uint64_t dpa_offset = 0;
1159 AddressSpace *as = NULL;
1160 int res;
1161
1162 res = cxl_type3_hpa_to_as_and_dpa(ct3d, host_addr, size,
1163 &as, &dpa_offset);
1164 if (res) {
1165 return MEMTX_ERROR;
1166 }
1167
1168 if (cxl_dev_media_disabled(&ct3d->cxl_dstate)) {
1169 qemu_guest_getrandom_nofail(data, size);
1170 return MEMTX_OK;
1171 }
1172
1173 return address_space_read(as, dpa_offset, attrs, data, size);
1174 }
1175
cxl_type3_write(PCIDevice * d,hwaddr host_addr,uint64_t data,unsigned size,MemTxAttrs attrs)1176 MemTxResult cxl_type3_write(PCIDevice *d, hwaddr host_addr, uint64_t data,
1177 unsigned size, MemTxAttrs attrs)
1178 {
1179 CXLType3Dev *ct3d = CXL_TYPE3(d);
1180 uint64_t dpa_offset = 0;
1181 AddressSpace *as = NULL;
1182 int res;
1183
1184 res = cxl_type3_hpa_to_as_and_dpa(ct3d, host_addr, size,
1185 &as, &dpa_offset);
1186 if (res) {
1187 return MEMTX_ERROR;
1188 }
1189
1190 if (cxl_dev_media_disabled(&ct3d->cxl_dstate)) {
1191 return MEMTX_OK;
1192 }
1193
1194 return address_space_write(as, dpa_offset, attrs, &data, size);
1195 }
1196
ct3d_reset(DeviceState * dev)1197 static void ct3d_reset(DeviceState *dev)
1198 {
1199 CXLType3Dev *ct3d = CXL_TYPE3(dev);
1200 uint32_t *reg_state = ct3d->cxl_cstate.crb.cache_mem_registers;
1201 uint32_t *write_msk = ct3d->cxl_cstate.crb.cache_mem_regs_write_mask;
1202
1203 cxl_component_register_init_common(reg_state, write_msk, CXL2_TYPE3_DEVICE);
1204 cxl_device_register_init_t3(ct3d);
1205
1206 /*
1207 * Bring up an endpoint to target with MCTP over VDM.
1208 * This device is emulating an MLD with single LD for now.
1209 */
1210 cxl_initialize_t3_fm_owned_ld_mctpcci(&ct3d->vdm_fm_owned_ld_mctp_cci,
1211 DEVICE(ct3d), DEVICE(ct3d),
1212 512); /* Max payload made up */
1213 cxl_initialize_t3_ld_cci(&ct3d->ld0_cci, DEVICE(ct3d), DEVICE(ct3d),
1214 512); /* Max payload made up */
1215
1216 }
1217
1218 static Property ct3_props[] = {
1219 DEFINE_PROP_LINK("memdev", CXLType3Dev, hostmem, TYPE_MEMORY_BACKEND,
1220 HostMemoryBackend *), /* for backward compatibility */
1221 DEFINE_PROP_LINK("persistent-memdev", CXLType3Dev, hostpmem,
1222 TYPE_MEMORY_BACKEND, HostMemoryBackend *),
1223 DEFINE_PROP_LINK("volatile-memdev", CXLType3Dev, hostvmem,
1224 TYPE_MEMORY_BACKEND, HostMemoryBackend *),
1225 DEFINE_PROP_LINK("lsa", CXLType3Dev, lsa, TYPE_MEMORY_BACKEND,
1226 HostMemoryBackend *),
1227 DEFINE_PROP_UINT64("sn", CXLType3Dev, sn, UI64_NULL),
1228 DEFINE_PROP_STRING("cdat", CXLType3Dev, cxl_cstate.cdat.filename),
1229 DEFINE_PROP_UINT8("num-dc-regions", CXLType3Dev, dc.num_regions, 0),
1230 DEFINE_PROP_LINK("volatile-dc-memdev", CXLType3Dev, dc.host_dc,
1231 TYPE_MEMORY_BACKEND, HostMemoryBackend *),
1232 DEFINE_PROP_END_OF_LIST(),
1233 };
1234
get_lsa_size(CXLType3Dev * ct3d)1235 static uint64_t get_lsa_size(CXLType3Dev *ct3d)
1236 {
1237 MemoryRegion *mr;
1238
1239 if (!ct3d->lsa) {
1240 return 0;
1241 }
1242
1243 mr = host_memory_backend_get_memory(ct3d->lsa);
1244 return memory_region_size(mr);
1245 }
1246
validate_lsa_access(MemoryRegion * mr,uint64_t size,uint64_t offset)1247 static void validate_lsa_access(MemoryRegion *mr, uint64_t size,
1248 uint64_t offset)
1249 {
1250 assert(offset + size <= memory_region_size(mr));
1251 assert(offset + size > offset);
1252 }
1253
get_lsa(CXLType3Dev * ct3d,void * buf,uint64_t size,uint64_t offset)1254 static uint64_t get_lsa(CXLType3Dev *ct3d, void *buf, uint64_t size,
1255 uint64_t offset)
1256 {
1257 MemoryRegion *mr;
1258 void *lsa;
1259
1260 if (!ct3d->lsa) {
1261 return 0;
1262 }
1263
1264 mr = host_memory_backend_get_memory(ct3d->lsa);
1265 validate_lsa_access(mr, size, offset);
1266
1267 lsa = memory_region_get_ram_ptr(mr) + offset;
1268 memcpy(buf, lsa, size);
1269
1270 return size;
1271 }
1272
set_lsa(CXLType3Dev * ct3d,const void * buf,uint64_t size,uint64_t offset)1273 static void set_lsa(CXLType3Dev *ct3d, const void *buf, uint64_t size,
1274 uint64_t offset)
1275 {
1276 MemoryRegion *mr;
1277 void *lsa;
1278
1279 if (!ct3d->lsa) {
1280 return;
1281 }
1282
1283 mr = host_memory_backend_get_memory(ct3d->lsa);
1284 validate_lsa_access(mr, size, offset);
1285
1286 lsa = memory_region_get_ram_ptr(mr) + offset;
1287 memcpy(lsa, buf, size);
1288 memory_region_set_dirty(mr, offset, size);
1289
1290 /*
1291 * Just like the PMEM, if the guest is not allowed to exit gracefully, label
1292 * updates will get lost.
1293 */
1294 }
1295
set_cacheline(CXLType3Dev * ct3d,uint64_t dpa_offset,uint8_t * data)1296 static bool set_cacheline(CXLType3Dev *ct3d, uint64_t dpa_offset, uint8_t *data)
1297 {
1298 MemoryRegion *vmr = NULL, *pmr = NULL, *dc_mr = NULL;
1299 AddressSpace *as;
1300 uint64_t vmr_size = 0, pmr_size = 0, dc_size = 0;
1301
1302 if (ct3d->hostvmem) {
1303 vmr = host_memory_backend_get_memory(ct3d->hostvmem);
1304 vmr_size = memory_region_size(vmr);
1305 }
1306 if (ct3d->hostpmem) {
1307 pmr = host_memory_backend_get_memory(ct3d->hostpmem);
1308 pmr_size = memory_region_size(pmr);
1309 }
1310 if (ct3d->dc.host_dc) {
1311 dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
1312 dc_size = memory_region_size(dc_mr);
1313 }
1314
1315 if (!vmr && !pmr && !dc_mr) {
1316 return false;
1317 }
1318
1319 if (dpa_offset + CXL_CACHE_LINE_SIZE > vmr_size + pmr_size + dc_size) {
1320 return false;
1321 }
1322
1323 if (dpa_offset < vmr_size) {
1324 as = &ct3d->hostvmem_as;
1325 } else if (dpa_offset < vmr_size + pmr_size) {
1326 as = &ct3d->hostpmem_as;
1327 dpa_offset -= vmr_size;
1328 } else {
1329 as = &ct3d->dc.host_dc_as;
1330 dpa_offset -= (vmr_size + pmr_size);
1331 }
1332
1333 address_space_write(as, dpa_offset, MEMTXATTRS_UNSPECIFIED, data,
1334 CXL_CACHE_LINE_SIZE);
1335 return true;
1336 }
1337
cxl_set_poison_list_overflowed(CXLType3Dev * ct3d)1338 void cxl_set_poison_list_overflowed(CXLType3Dev *ct3d)
1339 {
1340 ct3d->poison_list_overflowed = true;
1341 ct3d->poison_list_overflow_ts =
1342 cxl_device_get_timestamp(&ct3d->cxl_dstate);
1343 }
1344
cxl_clear_poison_list_overflowed(CXLType3Dev * ct3d)1345 void cxl_clear_poison_list_overflowed(CXLType3Dev *ct3d)
1346 {
1347 ct3d->poison_list_overflowed = false;
1348 ct3d->poison_list_overflow_ts = 0;
1349 }
1350
qmp_cxl_inject_poison(const char * path,uint64_t start,uint64_t length,Error ** errp)1351 void qmp_cxl_inject_poison(const char *path, uint64_t start, uint64_t length,
1352 Error **errp)
1353 {
1354 Object *obj = object_resolve_path(path, NULL);
1355 CXLType3Dev *ct3d;
1356 CXLPoison *p;
1357
1358 if (length % 64) {
1359 error_setg(errp, "Poison injection must be in multiples of 64 bytes");
1360 return;
1361 }
1362 if (start % 64) {
1363 error_setg(errp, "Poison start address must be 64 byte aligned");
1364 return;
1365 }
1366 if (!obj) {
1367 error_setg(errp, "Unable to resolve path");
1368 return;
1369 }
1370 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1371 error_setg(errp, "Path does not point to a CXL type 3 device");
1372 return;
1373 }
1374
1375 ct3d = CXL_TYPE3(obj);
1376
1377 QLIST_FOREACH(p, &ct3d->poison_list, node) {
1378 if (((start >= p->start) && (start < p->start + p->length)) ||
1379 ((start + length > p->start) &&
1380 (start + length <= p->start + p->length))) {
1381 error_setg(errp,
1382 "Overlap with existing poisoned region not supported");
1383 return;
1384 }
1385 }
1386
1387 p = g_new0(CXLPoison, 1);
1388 p->length = length;
1389 p->start = start;
1390 /* Different from injected via the mbox */
1391 p->type = CXL_POISON_TYPE_INTERNAL;
1392
1393 if (ct3d->poison_list_cnt < CXL_POISON_LIST_LIMIT) {
1394 QLIST_INSERT_HEAD(&ct3d->poison_list, p, node);
1395 ct3d->poison_list_cnt++;
1396 } else {
1397 if (!ct3d->poison_list_overflowed) {
1398 cxl_set_poison_list_overflowed(ct3d);
1399 }
1400 QLIST_INSERT_HEAD(&ct3d->poison_list_bkp, p, node);
1401 }
1402 }
1403
1404 /* For uncorrectable errors include support for multiple header recording */
qmp_cxl_inject_uncorrectable_errors(const char * path,CXLUncorErrorRecordList * errors,Error ** errp)1405 void qmp_cxl_inject_uncorrectable_errors(const char *path,
1406 CXLUncorErrorRecordList *errors,
1407 Error **errp)
1408 {
1409 Object *obj = object_resolve_path(path, NULL);
1410 static PCIEAERErr err = {};
1411 CXLType3Dev *ct3d;
1412 CXLError *cxl_err;
1413 uint32_t *reg_state;
1414 uint32_t unc_err;
1415 bool first;
1416
1417 if (!obj) {
1418 error_setg(errp, "Unable to resolve path");
1419 return;
1420 }
1421
1422 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1423 error_setg(errp, "Path does not point to a CXL type 3 device");
1424 return;
1425 }
1426
1427 err.status = PCI_ERR_UNC_INTN;
1428 err.source_id = pci_requester_id(PCI_DEVICE(obj));
1429 err.flags = 0;
1430
1431 ct3d = CXL_TYPE3(obj);
1432
1433 first = QTAILQ_EMPTY(&ct3d->error_list);
1434 reg_state = ct3d->cxl_cstate.crb.cache_mem_registers;
1435 while (errors) {
1436 uint32List *header = errors->value->header;
1437 uint8_t header_count = 0;
1438 int cxl_err_code;
1439
1440 cxl_err_code = ct3d_qmp_uncor_err_to_cxl(errors->value->type);
1441 if (cxl_err_code < 0) {
1442 error_setg(errp, "Unknown error code");
1443 return;
1444 }
1445
1446 /* If the error is masked, nothing to do here */
1447 if (!((1 << cxl_err_code) &
1448 ~ldl_le_p(reg_state + R_CXL_RAS_UNC_ERR_MASK))) {
1449 errors = errors->next;
1450 continue;
1451 }
1452
1453 cxl_err = g_malloc0(sizeof(*cxl_err));
1454
1455 cxl_err->type = cxl_err_code;
1456 while (header && header_count < 32) {
1457 cxl_err->header[header_count++] = header->value;
1458 header = header->next;
1459 }
1460 if (header_count > 32) {
1461 error_setg(errp, "Header must be 32 DWORD or less");
1462 return;
1463 }
1464 QTAILQ_INSERT_TAIL(&ct3d->error_list, cxl_err, node);
1465
1466 errors = errors->next;
1467 }
1468
1469 if (first && !QTAILQ_EMPTY(&ct3d->error_list)) {
1470 uint32_t *cache_mem = ct3d->cxl_cstate.crb.cache_mem_registers;
1471 uint32_t capctrl = ldl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL);
1472 uint32_t *header_log = &cache_mem[R_CXL_RAS_ERR_HEADER0];
1473 int i;
1474
1475 cxl_err = QTAILQ_FIRST(&ct3d->error_list);
1476 for (i = 0; i < CXL_RAS_ERR_HEADER_NUM; i++) {
1477 stl_le_p(header_log + i, cxl_err->header[i]);
1478 }
1479
1480 capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL,
1481 FIRST_ERROR_POINTER, cxl_err->type);
1482 stl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL, capctrl);
1483 }
1484
1485 unc_err = 0;
1486 QTAILQ_FOREACH(cxl_err, &ct3d->error_list, node) {
1487 unc_err |= (1 << cxl_err->type);
1488 }
1489 if (!unc_err) {
1490 return;
1491 }
1492
1493 stl_le_p(reg_state + R_CXL_RAS_UNC_ERR_STATUS, unc_err);
1494 pcie_aer_inject_error(PCI_DEVICE(obj), &err);
1495
1496 return;
1497 }
1498
qmp_cxl_inject_correctable_error(const char * path,CxlCorErrorType type,Error ** errp)1499 void qmp_cxl_inject_correctable_error(const char *path, CxlCorErrorType type,
1500 Error **errp)
1501 {
1502 static PCIEAERErr err = {};
1503 Object *obj = object_resolve_path(path, NULL);
1504 CXLType3Dev *ct3d;
1505 uint32_t *reg_state;
1506 uint32_t cor_err;
1507 int cxl_err_type;
1508
1509 if (!obj) {
1510 error_setg(errp, "Unable to resolve path");
1511 return;
1512 }
1513 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1514 error_setg(errp, "Path does not point to a CXL type 3 device");
1515 return;
1516 }
1517
1518 err.status = PCI_ERR_COR_INTERNAL;
1519 err.source_id = pci_requester_id(PCI_DEVICE(obj));
1520 err.flags = PCIE_AER_ERR_IS_CORRECTABLE;
1521
1522 ct3d = CXL_TYPE3(obj);
1523 reg_state = ct3d->cxl_cstate.crb.cache_mem_registers;
1524 cor_err = ldl_le_p(reg_state + R_CXL_RAS_COR_ERR_STATUS);
1525
1526 cxl_err_type = ct3d_qmp_cor_err_to_cxl(type);
1527 if (cxl_err_type < 0) {
1528 error_setg(errp, "Invalid COR error");
1529 return;
1530 }
1531 /* If the error is masked, nothting to do here */
1532 if (!((1 << cxl_err_type) &
1533 ~ldl_le_p(reg_state + R_CXL_RAS_COR_ERR_MASK))) {
1534 return;
1535 }
1536
1537 cor_err |= (1 << cxl_err_type);
1538 stl_le_p(reg_state + R_CXL_RAS_COR_ERR_STATUS, cor_err);
1539
1540 pcie_aer_inject_error(PCI_DEVICE(obj), &err);
1541 }
1542
cxl_assign_event_header(CXLEventRecordHdr * hdr,const QemuUUID * uuid,uint32_t flags,uint8_t length,uint64_t timestamp)1543 static void cxl_assign_event_header(CXLEventRecordHdr *hdr,
1544 const QemuUUID *uuid, uint32_t flags,
1545 uint8_t length, uint64_t timestamp)
1546 {
1547 st24_le_p(&hdr->flags, flags);
1548 hdr->length = length;
1549 memcpy(&hdr->id, uuid, sizeof(hdr->id));
1550 stq_le_p(&hdr->timestamp, timestamp);
1551 }
1552
1553 static const QemuUUID gen_media_uuid = {
1554 .data = UUID(0xfbcd0a77, 0xc260, 0x417f,
1555 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6),
1556 };
1557
1558 static const QemuUUID dram_uuid = {
1559 .data = UUID(0x601dcbb3, 0x9c06, 0x4eab, 0xb8, 0xaf,
1560 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24),
1561 };
1562
1563 static const QemuUUID memory_module_uuid = {
1564 .data = UUID(0xfe927475, 0xdd59, 0x4339, 0xa5, 0x86,
1565 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74),
1566 };
1567
1568 #define CXL_GMER_VALID_CHANNEL BIT(0)
1569 #define CXL_GMER_VALID_RANK BIT(1)
1570 #define CXL_GMER_VALID_DEVICE BIT(2)
1571 #define CXL_GMER_VALID_COMPONENT BIT(3)
1572
ct3d_qmp_cxl_event_log_enc(CxlEventLog log)1573 static int ct3d_qmp_cxl_event_log_enc(CxlEventLog log)
1574 {
1575 switch (log) {
1576 case CXL_EVENT_LOG_INFORMATIONAL:
1577 return CXL_EVENT_TYPE_INFO;
1578 case CXL_EVENT_LOG_WARNING:
1579 return CXL_EVENT_TYPE_WARN;
1580 case CXL_EVENT_LOG_FAILURE:
1581 return CXL_EVENT_TYPE_FAIL;
1582 case CXL_EVENT_LOG_FATAL:
1583 return CXL_EVENT_TYPE_FATAL;
1584 default:
1585 return -EINVAL;
1586 }
1587 }
1588 /* Component ID is device specific. Define this as a string. */
qmp_cxl_inject_general_media_event(const char * path,CxlEventLog log,uint8_t flags,uint64_t dpa,uint8_t descriptor,uint8_t type,uint8_t transaction_type,bool has_channel,uint8_t channel,bool has_rank,uint8_t rank,bool has_device,uint32_t device,const char * component_id,Error ** errp)1589 void qmp_cxl_inject_general_media_event(const char *path, CxlEventLog log,
1590 uint8_t flags, uint64_t dpa,
1591 uint8_t descriptor, uint8_t type,
1592 uint8_t transaction_type,
1593 bool has_channel, uint8_t channel,
1594 bool has_rank, uint8_t rank,
1595 bool has_device, uint32_t device,
1596 const char *component_id,
1597 Error **errp)
1598 {
1599 Object *obj = object_resolve_path(path, NULL);
1600 CXLEventGenMedia gem;
1601 CXLEventRecordHdr *hdr = &gem.hdr;
1602 CXLDeviceState *cxlds;
1603 CXLType3Dev *ct3d;
1604 uint16_t valid_flags = 0;
1605 uint8_t enc_log;
1606 int rc;
1607
1608 if (!obj) {
1609 error_setg(errp, "Unable to resolve path");
1610 return;
1611 }
1612 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1613 error_setg(errp, "Path does not point to a CXL type 3 device");
1614 return;
1615 }
1616 ct3d = CXL_TYPE3(obj);
1617 cxlds = &ct3d->cxl_dstate;
1618
1619 rc = ct3d_qmp_cxl_event_log_enc(log);
1620 if (rc < 0) {
1621 error_setg(errp, "Unhandled error log type");
1622 return;
1623 }
1624 enc_log = rc;
1625
1626 memset(&gem, 0, sizeof(gem));
1627 cxl_assign_event_header(hdr, &gen_media_uuid, flags, sizeof(gem),
1628 cxl_device_get_timestamp(&ct3d->cxl_dstate));
1629
1630 stq_le_p(&gem.phys_addr, dpa);
1631 gem.descriptor = descriptor;
1632 gem.type = type;
1633 gem.transaction_type = transaction_type;
1634
1635 if (has_channel) {
1636 gem.channel = channel;
1637 valid_flags |= CXL_GMER_VALID_CHANNEL;
1638 }
1639
1640 if (has_rank) {
1641 gem.rank = rank;
1642 valid_flags |= CXL_GMER_VALID_RANK;
1643 }
1644
1645 if (has_device) {
1646 st24_le_p(gem.device, device);
1647 valid_flags |= CXL_GMER_VALID_DEVICE;
1648 }
1649
1650 if (component_id) {
1651 strncpy((char *)gem.component_id, component_id,
1652 sizeof(gem.component_id) - 1);
1653 valid_flags |= CXL_GMER_VALID_COMPONENT;
1654 }
1655
1656 stw_le_p(&gem.validity_flags, valid_flags);
1657
1658 if (cxl_event_insert(cxlds, enc_log, (CXLEventRecordRaw *)&gem)) {
1659 cxl_event_irq_assert(ct3d);
1660 }
1661 }
1662
1663 #define CXL_DRAM_VALID_CHANNEL BIT(0)
1664 #define CXL_DRAM_VALID_RANK BIT(1)
1665 #define CXL_DRAM_VALID_NIBBLE_MASK BIT(2)
1666 #define CXL_DRAM_VALID_BANK_GROUP BIT(3)
1667 #define CXL_DRAM_VALID_BANK BIT(4)
1668 #define CXL_DRAM_VALID_ROW BIT(5)
1669 #define CXL_DRAM_VALID_COLUMN BIT(6)
1670 #define CXL_DRAM_VALID_CORRECTION_MASK BIT(7)
1671
qmp_cxl_inject_dram_event(const char * path,CxlEventLog log,uint8_t flags,uint64_t dpa,uint8_t descriptor,uint8_t type,uint8_t transaction_type,bool has_channel,uint8_t channel,bool has_rank,uint8_t rank,bool has_nibble_mask,uint32_t nibble_mask,bool has_bank_group,uint8_t bank_group,bool has_bank,uint8_t bank,bool has_row,uint32_t row,bool has_column,uint16_t column,bool has_correction_mask,uint64List * correction_mask,Error ** errp)1672 void qmp_cxl_inject_dram_event(const char *path, CxlEventLog log, uint8_t flags,
1673 uint64_t dpa, uint8_t descriptor,
1674 uint8_t type, uint8_t transaction_type,
1675 bool has_channel, uint8_t channel,
1676 bool has_rank, uint8_t rank,
1677 bool has_nibble_mask, uint32_t nibble_mask,
1678 bool has_bank_group, uint8_t bank_group,
1679 bool has_bank, uint8_t bank,
1680 bool has_row, uint32_t row,
1681 bool has_column, uint16_t column,
1682 bool has_correction_mask,
1683 uint64List *correction_mask,
1684 Error **errp)
1685 {
1686 Object *obj = object_resolve_path(path, NULL);
1687 CXLEventDram dram;
1688 CXLEventRecordHdr *hdr = &dram.hdr;
1689 CXLDeviceState *cxlds;
1690 CXLType3Dev *ct3d;
1691 uint16_t valid_flags = 0;
1692 uint8_t enc_log;
1693 int rc;
1694
1695 if (!obj) {
1696 error_setg(errp, "Unable to resolve path");
1697 return;
1698 }
1699 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1700 error_setg(errp, "Path does not point to a CXL type 3 device");
1701 return;
1702 }
1703 ct3d = CXL_TYPE3(obj);
1704 cxlds = &ct3d->cxl_dstate;
1705
1706 rc = ct3d_qmp_cxl_event_log_enc(log);
1707 if (rc < 0) {
1708 error_setg(errp, "Unhandled error log type");
1709 return;
1710 }
1711 enc_log = rc;
1712
1713 memset(&dram, 0, sizeof(dram));
1714 cxl_assign_event_header(hdr, &dram_uuid, flags, sizeof(dram),
1715 cxl_device_get_timestamp(&ct3d->cxl_dstate));
1716 stq_le_p(&dram.phys_addr, dpa);
1717 dram.descriptor = descriptor;
1718 dram.type = type;
1719 dram.transaction_type = transaction_type;
1720
1721 if (has_channel) {
1722 dram.channel = channel;
1723 valid_flags |= CXL_DRAM_VALID_CHANNEL;
1724 }
1725
1726 if (has_rank) {
1727 dram.rank = rank;
1728 valid_flags |= CXL_DRAM_VALID_RANK;
1729 }
1730
1731 if (has_nibble_mask) {
1732 st24_le_p(dram.nibble_mask, nibble_mask);
1733 valid_flags |= CXL_DRAM_VALID_NIBBLE_MASK;
1734 }
1735
1736 if (has_bank_group) {
1737 dram.bank_group = bank_group;
1738 valid_flags |= CXL_DRAM_VALID_BANK_GROUP;
1739 }
1740
1741 if (has_bank) {
1742 dram.bank = bank;
1743 valid_flags |= CXL_DRAM_VALID_BANK;
1744 }
1745
1746 if (has_row) {
1747 st24_le_p(dram.row, row);
1748 valid_flags |= CXL_DRAM_VALID_ROW;
1749 }
1750
1751 if (has_column) {
1752 stw_le_p(&dram.column, column);
1753 valid_flags |= CXL_DRAM_VALID_COLUMN;
1754 }
1755
1756 if (has_correction_mask) {
1757 int count = 0;
1758 while (correction_mask && count < 4) {
1759 stq_le_p(&dram.correction_mask[count],
1760 correction_mask->value);
1761 count++;
1762 correction_mask = correction_mask->next;
1763 }
1764 valid_flags |= CXL_DRAM_VALID_CORRECTION_MASK;
1765 }
1766
1767 stw_le_p(&dram.validity_flags, valid_flags);
1768
1769 if (cxl_event_insert(cxlds, enc_log, (CXLEventRecordRaw *)&dram)) {
1770 cxl_event_irq_assert(ct3d);
1771 }
1772 return;
1773 }
1774
qmp_cxl_inject_memory_module_event(const char * path,CxlEventLog log,uint8_t flags,uint8_t type,uint8_t health_status,uint8_t media_status,uint8_t additional_status,uint8_t life_used,int16_t temperature,uint32_t dirty_shutdown_count,uint32_t corrected_volatile_error_count,uint32_t corrected_persist_error_count,Error ** errp)1775 void qmp_cxl_inject_memory_module_event(const char *path, CxlEventLog log,
1776 uint8_t flags, uint8_t type,
1777 uint8_t health_status,
1778 uint8_t media_status,
1779 uint8_t additional_status,
1780 uint8_t life_used,
1781 int16_t temperature,
1782 uint32_t dirty_shutdown_count,
1783 uint32_t corrected_volatile_error_count,
1784 uint32_t corrected_persist_error_count,
1785 Error **errp)
1786 {
1787 Object *obj = object_resolve_path(path, NULL);
1788 CXLEventMemoryModule module;
1789 CXLEventRecordHdr *hdr = &module.hdr;
1790 CXLDeviceState *cxlds;
1791 CXLType3Dev *ct3d;
1792 uint8_t enc_log;
1793 int rc;
1794
1795 if (!obj) {
1796 error_setg(errp, "Unable to resolve path");
1797 return;
1798 }
1799 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1800 error_setg(errp, "Path does not point to a CXL type 3 device");
1801 return;
1802 }
1803 ct3d = CXL_TYPE3(obj);
1804 cxlds = &ct3d->cxl_dstate;
1805
1806 rc = ct3d_qmp_cxl_event_log_enc(log);
1807 if (rc < 0) {
1808 error_setg(errp, "Unhandled error log type");
1809 return;
1810 }
1811 enc_log = rc;
1812
1813 memset(&module, 0, sizeof(module));
1814 cxl_assign_event_header(hdr, &memory_module_uuid, flags, sizeof(module),
1815 cxl_device_get_timestamp(&ct3d->cxl_dstate));
1816
1817 module.type = type;
1818 module.health_status = health_status;
1819 module.media_status = media_status;
1820 module.additional_status = additional_status;
1821 module.life_used = life_used;
1822 stw_le_p(&module.temperature, temperature);
1823 stl_le_p(&module.dirty_shutdown_count, dirty_shutdown_count);
1824 stl_le_p(&module.corrected_volatile_error_count,
1825 corrected_volatile_error_count);
1826 stl_le_p(&module.corrected_persistent_error_count,
1827 corrected_persist_error_count);
1828
1829 if (cxl_event_insert(cxlds, enc_log, (CXLEventRecordRaw *)&module)) {
1830 cxl_event_irq_assert(ct3d);
1831 }
1832 }
1833
1834 /* CXL r3.1 Table 8-50: Dynamic Capacity Event Record */
1835 static const QemuUUID dynamic_capacity_uuid = {
1836 .data = UUID(0xca95afa7, 0xf183, 0x4018, 0x8c, 0x2f,
1837 0x95, 0x26, 0x8e, 0x10, 0x1a, 0x2a),
1838 };
1839
1840 typedef enum CXLDCEventType {
1841 DC_EVENT_ADD_CAPACITY = 0x0,
1842 DC_EVENT_RELEASE_CAPACITY = 0x1,
1843 DC_EVENT_FORCED_RELEASE_CAPACITY = 0x2,
1844 DC_EVENT_REGION_CONFIG_UPDATED = 0x3,
1845 DC_EVENT_ADD_CAPACITY_RSP = 0x4,
1846 DC_EVENT_CAPACITY_RELEASED = 0x5,
1847 } CXLDCEventType;
1848
1849 /*
1850 * Check whether the range [dpa, dpa + len - 1] has overlaps with extents in
1851 * the list.
1852 * Return value: return true if has overlaps; otherwise, return false
1853 */
cxl_extents_overlaps_dpa_range(CXLDCExtentList * list,uint64_t dpa,uint64_t len)1854 static bool cxl_extents_overlaps_dpa_range(CXLDCExtentList *list,
1855 uint64_t dpa, uint64_t len)
1856 {
1857 CXLDCExtent *ent;
1858 Range range1, range2;
1859
1860 if (!list) {
1861 return false;
1862 }
1863
1864 range_init_nofail(&range1, dpa, len);
1865 QTAILQ_FOREACH(ent, list, node) {
1866 range_init_nofail(&range2, ent->start_dpa, ent->len);
1867 if (range_overlaps_range(&range1, &range2)) {
1868 return true;
1869 }
1870 }
1871 return false;
1872 }
1873
1874 /*
1875 * Check whether the range [dpa, dpa + len - 1] is contained by extents in
1876 * the list.
1877 * Will check multiple extents containment once superset release is added.
1878 * Return value: return true if range is contained; otherwise, return false
1879 */
cxl_extents_contains_dpa_range(CXLDCExtentList * list,uint64_t dpa,uint64_t len)1880 bool cxl_extents_contains_dpa_range(CXLDCExtentList *list,
1881 uint64_t dpa, uint64_t len)
1882 {
1883 CXLDCExtent *ent;
1884 Range range1, range2;
1885
1886 if (!list) {
1887 return false;
1888 }
1889
1890 range_init_nofail(&range1, dpa, len);
1891 QTAILQ_FOREACH(ent, list, node) {
1892 range_init_nofail(&range2, ent->start_dpa, ent->len);
1893 if (range_contains_range(&range2, &range1)) {
1894 return true;
1895 }
1896 }
1897 return false;
1898 }
1899
cxl_extent_groups_overlaps_dpa_range(CXLDCExtentGroupList * list,uint64_t dpa,uint64_t len)1900 static bool cxl_extent_groups_overlaps_dpa_range(CXLDCExtentGroupList *list,
1901 uint64_t dpa, uint64_t len)
1902 {
1903 CXLDCExtentGroup *group;
1904
1905 if (!list) {
1906 return false;
1907 }
1908
1909 QTAILQ_FOREACH(group, list, node) {
1910 if (cxl_extents_overlaps_dpa_range(&group->list, dpa, len)) {
1911 return true;
1912 }
1913 }
1914 return false;
1915 }
1916
1917 /*
1918 * The main function to process dynamic capacity event with extent list.
1919 * Currently DC extents add/release requests are processed.
1920 */
qmp_cxl_process_dynamic_capacity_prescriptive(const char * path,uint16_t hid,CXLDCEventType type,uint8_t rid,CxlDynamicCapacityExtentList * records,Error ** errp)1921 static void qmp_cxl_process_dynamic_capacity_prescriptive(const char *path,
1922 uint16_t hid, CXLDCEventType type, uint8_t rid,
1923 CxlDynamicCapacityExtentList *records, Error **errp)
1924 {
1925 Object *obj;
1926 CXLEventDynamicCapacity dCap = {};
1927 CXLEventRecordHdr *hdr = &dCap.hdr;
1928 CXLType3Dev *dcd;
1929 uint8_t flags = 1 << CXL_EVENT_TYPE_INFO;
1930 uint32_t num_extents = 0;
1931 CxlDynamicCapacityExtentList *list;
1932 CXLDCExtentGroup *group = NULL;
1933 g_autofree CXLDCExtentRaw *extents = NULL;
1934 uint8_t enc_log = CXL_EVENT_TYPE_DYNAMIC_CAP;
1935 uint64_t dpa, offset, len, block_size;
1936 g_autofree unsigned long *blk_bitmap = NULL;
1937 int i;
1938
1939 obj = object_resolve_path_type(path, TYPE_CXL_TYPE3, NULL);
1940 if (!obj) {
1941 error_setg(errp, "Unable to resolve CXL type 3 device");
1942 return;
1943 }
1944
1945 dcd = CXL_TYPE3(obj);
1946 if (!dcd->dc.num_regions) {
1947 error_setg(errp, "No dynamic capacity support from the device");
1948 return;
1949 }
1950
1951
1952 if (rid >= dcd->dc.num_regions) {
1953 error_setg(errp, "region id is too large");
1954 return;
1955 }
1956 block_size = dcd->dc.regions[rid].block_size;
1957 blk_bitmap = bitmap_new(dcd->dc.regions[rid].len / block_size);
1958
1959 /* Sanity check and count the extents */
1960 list = records;
1961 while (list) {
1962 offset = list->value->offset;
1963 len = list->value->len;
1964 dpa = offset + dcd->dc.regions[rid].base;
1965
1966 if (len == 0) {
1967 error_setg(errp, "extent with 0 length is not allowed");
1968 return;
1969 }
1970
1971 if (offset % block_size || len % block_size) {
1972 error_setg(errp, "dpa or len is not aligned to region block size");
1973 return;
1974 }
1975
1976 if (offset + len > dcd->dc.regions[rid].len) {
1977 error_setg(errp, "extent range is beyond the region end");
1978 return;
1979 }
1980
1981 /* No duplicate or overlapped extents are allowed */
1982 if (test_any_bits_set(blk_bitmap, offset / block_size,
1983 len / block_size)) {
1984 error_setg(errp, "duplicate or overlapped extents are detected");
1985 return;
1986 }
1987 bitmap_set(blk_bitmap, offset / block_size, len / block_size);
1988
1989 if (type == DC_EVENT_RELEASE_CAPACITY) {
1990 if (cxl_extent_groups_overlaps_dpa_range(&dcd->dc.extents_pending,
1991 dpa, len)) {
1992 error_setg(errp,
1993 "cannot release extent with pending DPA range");
1994 return;
1995 }
1996 if (!ct3_test_region_block_backed(dcd, dpa, len)) {
1997 error_setg(errp,
1998 "cannot release extent with non-existing DPA range");
1999 return;
2000 }
2001 } else if (type == DC_EVENT_ADD_CAPACITY) {
2002 if (cxl_extents_overlaps_dpa_range(&dcd->dc.extents, dpa, len)) {
2003 error_setg(errp,
2004 "cannot add DPA already accessible to the same LD");
2005 return;
2006 }
2007 if (cxl_extent_groups_overlaps_dpa_range(&dcd->dc.extents_pending,
2008 dpa, len)) {
2009 error_setg(errp,
2010 "cannot add DPA again while still pending");
2011 return;
2012 }
2013 }
2014 list = list->next;
2015 num_extents++;
2016 }
2017
2018 /* Create extent list for event being passed to host */
2019 i = 0;
2020 list = records;
2021 extents = g_new0(CXLDCExtentRaw, num_extents);
2022 while (list) {
2023 offset = list->value->offset;
2024 len = list->value->len;
2025 dpa = dcd->dc.regions[rid].base + offset;
2026
2027 extents[i].start_dpa = dpa;
2028 extents[i].len = len;
2029 memset(extents[i].tag, 0, 0x10);
2030 extents[i].shared_seq = 0;
2031 if (type == DC_EVENT_ADD_CAPACITY) {
2032 group = cxl_insert_extent_to_extent_group(group,
2033 extents[i].start_dpa,
2034 extents[i].len,
2035 extents[i].tag,
2036 extents[i].shared_seq);
2037 }
2038
2039 list = list->next;
2040 i++;
2041 }
2042 if (group) {
2043 cxl_extent_group_list_insert_tail(&dcd->dc.extents_pending, group);
2044 }
2045
2046 /*
2047 * CXL r3.1 section 8.2.9.2.1.6: Dynamic Capacity Event Record
2048 *
2049 * All Dynamic Capacity event records shall set the Event Record Severity
2050 * field in the Common Event Record Format to Informational Event. All
2051 * Dynamic Capacity related events shall be logged in the Dynamic Capacity
2052 * Event Log.
2053 */
2054 cxl_assign_event_header(hdr, &dynamic_capacity_uuid, flags, sizeof(dCap),
2055 cxl_device_get_timestamp(&dcd->cxl_dstate));
2056
2057 dCap.type = type;
2058 /* FIXME: for now, validity flag is cleared */
2059 dCap.validity_flags = 0;
2060 stw_le_p(&dCap.host_id, hid);
2061 /* only valid for DC_REGION_CONFIG_UPDATED event */
2062 dCap.updated_region_id = 0;
2063 dCap.flags = 0;
2064 for (i = 0; i < num_extents; i++) {
2065 memcpy(&dCap.dynamic_capacity_extent, &extents[i],
2066 sizeof(CXLDCExtentRaw));
2067
2068 if (i < num_extents - 1) {
2069 /* Set "More" flag */
2070 dCap.flags |= BIT(0);
2071 }
2072
2073 if (cxl_event_insert(&dcd->cxl_dstate, enc_log,
2074 (CXLEventRecordRaw *)&dCap)) {
2075 cxl_event_irq_assert(dcd);
2076 }
2077 }
2078 }
2079
qmp_cxl_add_dynamic_capacity(const char * path,uint16_t host_id,CxlExtentSelectionPolicy sel_policy,uint8_t region,const char * tag,CxlDynamicCapacityExtentList * extents,Error ** errp)2080 void qmp_cxl_add_dynamic_capacity(const char *path, uint16_t host_id,
2081 CxlExtentSelectionPolicy sel_policy,
2082 uint8_t region, const char *tag,
2083 CxlDynamicCapacityExtentList *extents,
2084 Error **errp)
2085 {
2086 switch (sel_policy) {
2087 case CXL_EXTENT_SELECTION_POLICY_PRESCRIPTIVE:
2088 qmp_cxl_process_dynamic_capacity_prescriptive(path, host_id,
2089 DC_EVENT_ADD_CAPACITY,
2090 region, extents, errp);
2091 return;
2092 default:
2093 error_setg(errp, "Selection policy not supported");
2094 return;
2095 }
2096 }
2097
qmp_cxl_release_dynamic_capacity(const char * path,uint16_t host_id,CxlExtentRemovalPolicy removal_policy,bool has_forced_removal,bool forced_removal,bool has_sanitize_on_release,bool sanitize_on_release,uint8_t region,const char * tag,CxlDynamicCapacityExtentList * extents,Error ** errp)2098 void qmp_cxl_release_dynamic_capacity(const char *path, uint16_t host_id,
2099 CxlExtentRemovalPolicy removal_policy,
2100 bool has_forced_removal,
2101 bool forced_removal,
2102 bool has_sanitize_on_release,
2103 bool sanitize_on_release,
2104 uint8_t region,
2105 const char *tag,
2106 CxlDynamicCapacityExtentList *extents,
2107 Error **errp)
2108 {
2109 CXLDCEventType type = DC_EVENT_RELEASE_CAPACITY;
2110
2111 if (has_forced_removal && forced_removal) {
2112 /* TODO: enable forced removal in the future */
2113 type = DC_EVENT_FORCED_RELEASE_CAPACITY;
2114 error_setg(errp, "Forced removal not supported yet");
2115 return;
2116 }
2117
2118 switch (removal_policy) {
2119 case CXL_EXTENT_REMOVAL_POLICY_PRESCRIPTIVE:
2120 qmp_cxl_process_dynamic_capacity_prescriptive(path, host_id, type,
2121 region, extents, errp);
2122 return;
2123 default:
2124 error_setg(errp, "Removal policy not supported");
2125 return;
2126 }
2127 }
2128
ct3_class_init(ObjectClass * oc,void * data)2129 static void ct3_class_init(ObjectClass *oc, void *data)
2130 {
2131 DeviceClass *dc = DEVICE_CLASS(oc);
2132 PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
2133 CXLType3Class *cvc = CXL_TYPE3_CLASS(oc);
2134
2135 pc->realize = ct3_realize;
2136 pc->exit = ct3_exit;
2137 pc->class_id = PCI_CLASS_MEMORY_CXL;
2138 pc->vendor_id = PCI_VENDOR_ID_INTEL;
2139 pc->device_id = 0xd93; /* LVF for now */
2140 pc->revision = 1;
2141
2142 pc->config_write = ct3d_config_write;
2143 pc->config_read = ct3d_config_read;
2144
2145 set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
2146 dc->desc = "CXL Memory Device (Type 3)";
2147 dc->reset = ct3d_reset;
2148 device_class_set_props(dc, ct3_props);
2149
2150 cvc->get_lsa_size = get_lsa_size;
2151 cvc->get_lsa = get_lsa;
2152 cvc->set_lsa = set_lsa;
2153 cvc->set_cacheline = set_cacheline;
2154 }
2155
2156 static const TypeInfo ct3d_info = {
2157 .name = TYPE_CXL_TYPE3,
2158 .parent = TYPE_PCI_DEVICE,
2159 .class_size = sizeof(struct CXLType3Class),
2160 .class_init = ct3_class_init,
2161 .instance_size = sizeof(CXLType3Dev),
2162 .interfaces = (InterfaceInfo[]) {
2163 { INTERFACE_CXL_DEVICE },
2164 { INTERFACE_PCIE_DEVICE },
2165 {}
2166 },
2167 };
2168
ct3d_registers(void)2169 static void ct3d_registers(void)
2170 {
2171 type_register_static(&ct3d_info);
2172 }
2173
2174 type_init(ct3d_registers);
2175