1 /*
2 * CXL Type 3 (memory expander) device
3 *
4 * Copyright(C) 2020 Intel Corporation.
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2. See the
7 * COPYING file in the top-level directory.
8 *
9 * SPDX-License-Identifier: GPL-v2-only
10 */
11
12 #include "qemu/osdep.h"
13 #include "qemu/units.h"
14 #include "qemu/error-report.h"
15 #include "qapi/qapi-commands-cxl.h"
16 #include "hw/mem/memory-device.h"
17 #include "hw/mem/pc-dimm.h"
18 #include "hw/pci/pci.h"
19 #include "hw/qdev-properties.h"
20 #include "hw/qdev-properties-system.h"
21 #include "qapi/error.h"
22 #include "qemu/log.h"
23 #include "qemu/module.h"
24 #include "qemu/pmem.h"
25 #include "qemu/range.h"
26 #include "qemu/rcu.h"
27 #include "qemu/guest-random.h"
28 #include "sysemu/hostmem.h"
29 #include "sysemu/numa.h"
30 #include "hw/cxl/cxl.h"
31 #include "hw/pci/msix.h"
32
33 #define DWORD_BYTE 4
34 #define CXL_CAPACITY_MULTIPLIER (256 * MiB)
35
36 /* Default CDAT entries for a memory region */
37 enum {
38 CT3_CDAT_DSMAS,
39 CT3_CDAT_DSLBIS0,
40 CT3_CDAT_DSLBIS1,
41 CT3_CDAT_DSLBIS2,
42 CT3_CDAT_DSLBIS3,
43 CT3_CDAT_DSEMTS,
44 CT3_CDAT_NUM_ENTRIES
45 };
46
ct3_build_cdat_entries_for_mr(CDATSubHeader ** cdat_table,int dsmad_handle,uint64_t size,bool is_pmem,bool is_dynamic,uint64_t dpa_base)47 static void ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table,
48 int dsmad_handle, uint64_t size,
49 bool is_pmem, bool is_dynamic,
50 uint64_t dpa_base)
51 {
52 CDATDsmas *dsmas;
53 CDATDslbis *dslbis0;
54 CDATDslbis *dslbis1;
55 CDATDslbis *dslbis2;
56 CDATDslbis *dslbis3;
57 CDATDsemts *dsemts;
58
59 dsmas = g_malloc(sizeof(*dsmas));
60 *dsmas = (CDATDsmas) {
61 .header = {
62 .type = CDAT_TYPE_DSMAS,
63 .length = sizeof(*dsmas),
64 },
65 .DSMADhandle = dsmad_handle,
66 .flags = (is_pmem ? CDAT_DSMAS_FLAG_NV : 0) |
67 (is_dynamic ? CDAT_DSMAS_FLAG_DYNAMIC_CAP : 0),
68 .DPA_base = dpa_base,
69 .DPA_length = size,
70 };
71
72 /* For now, no memory side cache, plausiblish numbers */
73 dslbis0 = g_malloc(sizeof(*dslbis0));
74 *dslbis0 = (CDATDslbis) {
75 .header = {
76 .type = CDAT_TYPE_DSLBIS,
77 .length = sizeof(*dslbis0),
78 },
79 .handle = dsmad_handle,
80 .flags = HMAT_LB_MEM_MEMORY,
81 .data_type = HMAT_LB_DATA_READ_LATENCY,
82 .entry_base_unit = 10000, /* 10ns base */
83 .entry[0] = 15, /* 150ns */
84 };
85
86 dslbis1 = g_malloc(sizeof(*dslbis1));
87 *dslbis1 = (CDATDslbis) {
88 .header = {
89 .type = CDAT_TYPE_DSLBIS,
90 .length = sizeof(*dslbis1),
91 },
92 .handle = dsmad_handle,
93 .flags = HMAT_LB_MEM_MEMORY,
94 .data_type = HMAT_LB_DATA_WRITE_LATENCY,
95 .entry_base_unit = 10000,
96 .entry[0] = 25, /* 250ns */
97 };
98
99 dslbis2 = g_malloc(sizeof(*dslbis2));
100 *dslbis2 = (CDATDslbis) {
101 .header = {
102 .type = CDAT_TYPE_DSLBIS,
103 .length = sizeof(*dslbis2),
104 },
105 .handle = dsmad_handle,
106 .flags = HMAT_LB_MEM_MEMORY,
107 .data_type = HMAT_LB_DATA_READ_BANDWIDTH,
108 .entry_base_unit = 1000, /* GB/s */
109 .entry[0] = 16,
110 };
111
112 dslbis3 = g_malloc(sizeof(*dslbis3));
113 *dslbis3 = (CDATDslbis) {
114 .header = {
115 .type = CDAT_TYPE_DSLBIS,
116 .length = sizeof(*dslbis3),
117 },
118 .handle = dsmad_handle,
119 .flags = HMAT_LB_MEM_MEMORY,
120 .data_type = HMAT_LB_DATA_WRITE_BANDWIDTH,
121 .entry_base_unit = 1000, /* GB/s */
122 .entry[0] = 16,
123 };
124
125 dsemts = g_malloc(sizeof(*dsemts));
126 *dsemts = (CDATDsemts) {
127 .header = {
128 .type = CDAT_TYPE_DSEMTS,
129 .length = sizeof(*dsemts),
130 },
131 .DSMAS_handle = dsmad_handle,
132 /*
133 * NV: Reserved - the non volatile from DSMAS matters
134 * V: EFI_MEMORY_SP
135 */
136 .EFI_memory_type_attr = is_pmem ? 2 : 1,
137 .DPA_offset = 0,
138 .DPA_length = size,
139 };
140
141 /* Header always at start of structure */
142 cdat_table[CT3_CDAT_DSMAS] = (CDATSubHeader *)dsmas;
143 cdat_table[CT3_CDAT_DSLBIS0] = (CDATSubHeader *)dslbis0;
144 cdat_table[CT3_CDAT_DSLBIS1] = (CDATSubHeader *)dslbis1;
145 cdat_table[CT3_CDAT_DSLBIS2] = (CDATSubHeader *)dslbis2;
146 cdat_table[CT3_CDAT_DSLBIS3] = (CDATSubHeader *)dslbis3;
147 cdat_table[CT3_CDAT_DSEMTS] = (CDATSubHeader *)dsemts;
148 }
149
ct3_build_cdat_table(CDATSubHeader *** cdat_table,void * priv)150 static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv)
151 {
152 g_autofree CDATSubHeader **table = NULL;
153 CXLType3Dev *ct3d = priv;
154 MemoryRegion *volatile_mr = NULL, *nonvolatile_mr = NULL;
155 MemoryRegion *dc_mr = NULL;
156 uint64_t vmr_size = 0, pmr_size = 0;
157 int dsmad_handle = 0;
158 int cur_ent = 0;
159 int len = 0;
160
161 if (!ct3d->hostpmem && !ct3d->hostvmem && !ct3d->dc.num_regions) {
162 return 0;
163 }
164
165 if (ct3d->hostvmem) {
166 volatile_mr = host_memory_backend_get_memory(ct3d->hostvmem);
167 if (!volatile_mr) {
168 return -EINVAL;
169 }
170 len += CT3_CDAT_NUM_ENTRIES;
171 vmr_size = memory_region_size(volatile_mr);
172 }
173
174 if (ct3d->hostpmem) {
175 nonvolatile_mr = host_memory_backend_get_memory(ct3d->hostpmem);
176 if (!nonvolatile_mr) {
177 return -EINVAL;
178 }
179 len += CT3_CDAT_NUM_ENTRIES;
180 pmr_size = memory_region_size(nonvolatile_mr);
181 }
182
183 if (ct3d->dc.num_regions) {
184 if (!ct3d->dc.host_dc) {
185 return -EINVAL;
186 }
187 dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
188 if (!dc_mr) {
189 return -EINVAL;
190 }
191 len += CT3_CDAT_NUM_ENTRIES * ct3d->dc.num_regions;
192 }
193
194 table = g_malloc0(len * sizeof(*table));
195
196 /* Now fill them in */
197 if (volatile_mr) {
198 ct3_build_cdat_entries_for_mr(table, dsmad_handle++, vmr_size,
199 false, false, 0);
200 cur_ent = CT3_CDAT_NUM_ENTRIES;
201 }
202
203 if (nonvolatile_mr) {
204 uint64_t base = vmr_size;
205 ct3_build_cdat_entries_for_mr(&(table[cur_ent]), dsmad_handle++,
206 pmr_size, true, false, base);
207 cur_ent += CT3_CDAT_NUM_ENTRIES;
208 }
209
210 if (dc_mr) {
211 int i;
212 uint64_t region_base = vmr_size + pmr_size;
213
214 /*
215 * We assume the dynamic capacity to be volatile for now.
216 * Non-volatile dynamic capacity will be added if needed in the
217 * future.
218 */
219 for (i = 0; i < ct3d->dc.num_regions; i++) {
220 ct3_build_cdat_entries_for_mr(&(table[cur_ent]),
221 dsmad_handle++,
222 ct3d->dc.regions[i].len,
223 false, true, region_base);
224 ct3d->dc.regions[i].dsmadhandle = dsmad_handle - 1;
225
226 cur_ent += CT3_CDAT_NUM_ENTRIES;
227 region_base += ct3d->dc.regions[i].len;
228 }
229 }
230
231 assert(len == cur_ent);
232
233 *cdat_table = g_steal_pointer(&table);
234
235 return len;
236 }
237
ct3_free_cdat_table(CDATSubHeader ** cdat_table,int num,void * priv)238 static void ct3_free_cdat_table(CDATSubHeader **cdat_table, int num, void *priv)
239 {
240 int i;
241
242 for (i = 0; i < num; i++) {
243 g_free(cdat_table[i]);
244 }
245 g_free(cdat_table);
246 }
247
cxl_doe_cdat_rsp(DOECap * doe_cap)248 static bool cxl_doe_cdat_rsp(DOECap *doe_cap)
249 {
250 CDATObject *cdat = &CXL_TYPE3(doe_cap->pdev)->cxl_cstate.cdat;
251 uint16_t ent;
252 void *base;
253 uint32_t len;
254 CDATReq *req = pcie_doe_get_write_mbox_ptr(doe_cap);
255 CDATRsp rsp;
256
257 assert(cdat->entry_len);
258
259 /* Discard if request length mismatched */
260 if (pcie_doe_get_obj_len(req) <
261 DIV_ROUND_UP(sizeof(CDATReq), DWORD_BYTE)) {
262 return false;
263 }
264
265 ent = req->entry_handle;
266 base = cdat->entry[ent].base;
267 len = cdat->entry[ent].length;
268
269 rsp = (CDATRsp) {
270 .header = {
271 .vendor_id = CXL_VENDOR_ID,
272 .data_obj_type = CXL_DOE_TABLE_ACCESS,
273 .reserved = 0x0,
274 .length = DIV_ROUND_UP((sizeof(rsp) + len), DWORD_BYTE),
275 },
276 .rsp_code = CXL_DOE_TAB_RSP,
277 .table_type = CXL_DOE_TAB_TYPE_CDAT,
278 .entry_handle = (ent < cdat->entry_len - 1) ?
279 ent + 1 : CXL_DOE_TAB_ENT_MAX,
280 };
281
282 memcpy(doe_cap->read_mbox, &rsp, sizeof(rsp));
283 memcpy(doe_cap->read_mbox + DIV_ROUND_UP(sizeof(rsp), DWORD_BYTE),
284 base, len);
285
286 doe_cap->read_mbox_len += rsp.header.length;
287
288 return true;
289 }
290
ct3d_config_read(PCIDevice * pci_dev,uint32_t addr,int size)291 static uint32_t ct3d_config_read(PCIDevice *pci_dev, uint32_t addr, int size)
292 {
293 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev);
294 uint32_t val;
295
296 if (pcie_doe_read_config(&ct3d->doe_cdat, addr, size, &val)) {
297 return val;
298 }
299
300 return pci_default_read_config(pci_dev, addr, size);
301 }
302
ct3d_config_write(PCIDevice * pci_dev,uint32_t addr,uint32_t val,int size)303 static void ct3d_config_write(PCIDevice *pci_dev, uint32_t addr, uint32_t val,
304 int size)
305 {
306 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev);
307
308 pcie_doe_write_config(&ct3d->doe_cdat, addr, val, size);
309 pci_default_write_config(pci_dev, addr, val, size);
310 pcie_aer_write_config(pci_dev, addr, val, size);
311 }
312
313 /*
314 * Null value of all Fs suggested by IEEE RA guidelines for use of
315 * EU, OUI and CID
316 */
317 #define UI64_NULL ~(0ULL)
318
build_dvsecs(CXLType3Dev * ct3d)319 static void build_dvsecs(CXLType3Dev *ct3d)
320 {
321 CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;
322 uint8_t *dvsec;
323 uint32_t range1_size_hi, range1_size_lo,
324 range1_base_hi = 0, range1_base_lo = 0,
325 range2_size_hi = 0, range2_size_lo = 0,
326 range2_base_hi = 0, range2_base_lo = 0;
327
328 /*
329 * Volatile memory is mapped as (0x0)
330 * Persistent memory is mapped at (volatile->size)
331 */
332 if (ct3d->hostvmem) {
333 range1_size_hi = ct3d->hostvmem->size >> 32;
334 range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
335 (ct3d->hostvmem->size & 0xF0000000);
336 if (ct3d->hostpmem) {
337 range2_size_hi = ct3d->hostpmem->size >> 32;
338 range2_size_lo = (2 << 5) | (2 << 2) | 0x3 |
339 (ct3d->hostpmem->size & 0xF0000000);
340 }
341 } else if (ct3d->hostpmem) {
342 range1_size_hi = ct3d->hostpmem->size >> 32;
343 range1_size_lo = (2 << 5) | (2 << 2) | 0x3 |
344 (ct3d->hostpmem->size & 0xF0000000);
345 } else {
346 /*
347 * For DCD with no static memory, set memory active, memory class bits.
348 * No range is set.
349 */
350 range1_size_hi = 0;
351 range1_size_lo = (2 << 5) | (2 << 2) | 0x3;
352 }
353
354 dvsec = (uint8_t *)&(CXLDVSECDevice){
355 .cap = 0x1e,
356 .ctrl = 0x2,
357 .status2 = 0x2,
358 .range1_size_hi = range1_size_hi,
359 .range1_size_lo = range1_size_lo,
360 .range1_base_hi = range1_base_hi,
361 .range1_base_lo = range1_base_lo,
362 .range2_size_hi = range2_size_hi,
363 .range2_size_lo = range2_size_lo,
364 .range2_base_hi = range2_base_hi,
365 .range2_base_lo = range2_base_lo,
366 };
367 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE,
368 PCIE_CXL_DEVICE_DVSEC_LENGTH,
369 PCIE_CXL_DEVICE_DVSEC,
370 PCIE_CXL31_DEVICE_DVSEC_REVID, dvsec);
371
372 dvsec = (uint8_t *)&(CXLDVSECRegisterLocator){
373 .rsvd = 0,
374 .reg0_base_lo = RBI_COMPONENT_REG | CXL_COMPONENT_REG_BAR_IDX,
375 .reg0_base_hi = 0,
376 .reg1_base_lo = RBI_CXL_DEVICE_REG | CXL_DEVICE_REG_BAR_IDX,
377 .reg1_base_hi = 0,
378 };
379 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE,
380 REG_LOC_DVSEC_LENGTH, REG_LOC_DVSEC,
381 REG_LOC_DVSEC_REVID, dvsec);
382 dvsec = (uint8_t *)&(CXLDVSECDeviceGPF){
383 .phase2_duration = 0x603, /* 3 seconds */
384 .phase2_power = 0x33, /* 0x33 miliwatts */
385 };
386 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE,
387 GPF_DEVICE_DVSEC_LENGTH, GPF_DEVICE_DVSEC,
388 GPF_DEVICE_DVSEC_REVID, dvsec);
389
390 dvsec = (uint8_t *)&(CXLDVSECPortFlexBus){
391 .cap = 0x26, /* 68B, IO, Mem, non-MLD */
392 .ctrl = 0x02, /* IO always enabled */
393 .status = 0x26, /* same as capabilities */
394 .rcvd_mod_ts_data_phase1 = 0xef, /* WTF? */
395 };
396 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE,
397 PCIE_CXL3_FLEXBUS_PORT_DVSEC_LENGTH,
398 PCIE_FLEXBUS_PORT_DVSEC,
399 PCIE_CXL3_FLEXBUS_PORT_DVSEC_REVID, dvsec);
400 }
401
hdm_decoder_commit(CXLType3Dev * ct3d,int which)402 static void hdm_decoder_commit(CXLType3Dev *ct3d, int which)
403 {
404 int hdm_inc = R_CXL_HDM_DECODER1_BASE_LO - R_CXL_HDM_DECODER0_BASE_LO;
405 ComponentRegisters *cregs = &ct3d->cxl_cstate.crb;
406 uint32_t *cache_mem = cregs->cache_mem_registers;
407 uint32_t ctrl;
408
409 ctrl = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc);
410 /* TODO: Sanity checks that the decoder is possible */
411 ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, ERR, 0);
412 ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 1);
413
414 stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl);
415 }
416
hdm_decoder_uncommit(CXLType3Dev * ct3d,int which)417 static void hdm_decoder_uncommit(CXLType3Dev *ct3d, int which)
418 {
419 int hdm_inc = R_CXL_HDM_DECODER1_BASE_LO - R_CXL_HDM_DECODER0_BASE_LO;
420 ComponentRegisters *cregs = &ct3d->cxl_cstate.crb;
421 uint32_t *cache_mem = cregs->cache_mem_registers;
422 uint32_t ctrl;
423
424 ctrl = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc);
425
426 ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, ERR, 0);
427 ctrl = FIELD_DP32(ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED, 0);
428
429 stl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + which * hdm_inc, ctrl);
430 }
431
ct3d_qmp_uncor_err_to_cxl(CxlUncorErrorType qmp_err)432 static int ct3d_qmp_uncor_err_to_cxl(CxlUncorErrorType qmp_err)
433 {
434 switch (qmp_err) {
435 case CXL_UNCOR_ERROR_TYPE_CACHE_DATA_PARITY:
436 return CXL_RAS_UNC_ERR_CACHE_DATA_PARITY;
437 case CXL_UNCOR_ERROR_TYPE_CACHE_ADDRESS_PARITY:
438 return CXL_RAS_UNC_ERR_CACHE_ADDRESS_PARITY;
439 case CXL_UNCOR_ERROR_TYPE_CACHE_BE_PARITY:
440 return CXL_RAS_UNC_ERR_CACHE_BE_PARITY;
441 case CXL_UNCOR_ERROR_TYPE_CACHE_DATA_ECC:
442 return CXL_RAS_UNC_ERR_CACHE_DATA_ECC;
443 case CXL_UNCOR_ERROR_TYPE_MEM_DATA_PARITY:
444 return CXL_RAS_UNC_ERR_MEM_DATA_PARITY;
445 case CXL_UNCOR_ERROR_TYPE_MEM_ADDRESS_PARITY:
446 return CXL_RAS_UNC_ERR_MEM_ADDRESS_PARITY;
447 case CXL_UNCOR_ERROR_TYPE_MEM_BE_PARITY:
448 return CXL_RAS_UNC_ERR_MEM_BE_PARITY;
449 case CXL_UNCOR_ERROR_TYPE_MEM_DATA_ECC:
450 return CXL_RAS_UNC_ERR_MEM_DATA_ECC;
451 case CXL_UNCOR_ERROR_TYPE_REINIT_THRESHOLD:
452 return CXL_RAS_UNC_ERR_REINIT_THRESHOLD;
453 case CXL_UNCOR_ERROR_TYPE_RSVD_ENCODING:
454 return CXL_RAS_UNC_ERR_RSVD_ENCODING;
455 case CXL_UNCOR_ERROR_TYPE_POISON_RECEIVED:
456 return CXL_RAS_UNC_ERR_POISON_RECEIVED;
457 case CXL_UNCOR_ERROR_TYPE_RECEIVER_OVERFLOW:
458 return CXL_RAS_UNC_ERR_RECEIVER_OVERFLOW;
459 case CXL_UNCOR_ERROR_TYPE_INTERNAL:
460 return CXL_RAS_UNC_ERR_INTERNAL;
461 case CXL_UNCOR_ERROR_TYPE_CXL_IDE_TX:
462 return CXL_RAS_UNC_ERR_CXL_IDE_TX;
463 case CXL_UNCOR_ERROR_TYPE_CXL_IDE_RX:
464 return CXL_RAS_UNC_ERR_CXL_IDE_RX;
465 default:
466 return -EINVAL;
467 }
468 }
469
ct3d_qmp_cor_err_to_cxl(CxlCorErrorType qmp_err)470 static int ct3d_qmp_cor_err_to_cxl(CxlCorErrorType qmp_err)
471 {
472 switch (qmp_err) {
473 case CXL_COR_ERROR_TYPE_CACHE_DATA_ECC:
474 return CXL_RAS_COR_ERR_CACHE_DATA_ECC;
475 case CXL_COR_ERROR_TYPE_MEM_DATA_ECC:
476 return CXL_RAS_COR_ERR_MEM_DATA_ECC;
477 case CXL_COR_ERROR_TYPE_CRC_THRESHOLD:
478 return CXL_RAS_COR_ERR_CRC_THRESHOLD;
479 case CXL_COR_ERROR_TYPE_RETRY_THRESHOLD:
480 return CXL_RAS_COR_ERR_RETRY_THRESHOLD;
481 case CXL_COR_ERROR_TYPE_CACHE_POISON_RECEIVED:
482 return CXL_RAS_COR_ERR_CACHE_POISON_RECEIVED;
483 case CXL_COR_ERROR_TYPE_MEM_POISON_RECEIVED:
484 return CXL_RAS_COR_ERR_MEM_POISON_RECEIVED;
485 case CXL_COR_ERROR_TYPE_PHYSICAL:
486 return CXL_RAS_COR_ERR_PHYSICAL;
487 default:
488 return -EINVAL;
489 }
490 }
491
ct3d_reg_write(void * opaque,hwaddr offset,uint64_t value,unsigned size)492 static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value,
493 unsigned size)
494 {
495 CXLComponentState *cxl_cstate = opaque;
496 ComponentRegisters *cregs = &cxl_cstate->crb;
497 CXLType3Dev *ct3d = container_of(cxl_cstate, CXLType3Dev, cxl_cstate);
498 uint32_t *cache_mem = cregs->cache_mem_registers;
499 bool should_commit = false;
500 bool should_uncommit = false;
501 int which_hdm = -1;
502
503 assert(size == 4);
504 g_assert(offset < CXL2_COMPONENT_CM_REGION_SIZE);
505
506 switch (offset) {
507 case A_CXL_HDM_DECODER0_CTRL:
508 should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT);
509 should_uncommit = !should_commit;
510 which_hdm = 0;
511 break;
512 case A_CXL_HDM_DECODER1_CTRL:
513 should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT);
514 should_uncommit = !should_commit;
515 which_hdm = 1;
516 break;
517 case A_CXL_HDM_DECODER2_CTRL:
518 should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT);
519 should_uncommit = !should_commit;
520 which_hdm = 2;
521 break;
522 case A_CXL_HDM_DECODER3_CTRL:
523 should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT);
524 should_uncommit = !should_commit;
525 which_hdm = 3;
526 break;
527 case A_CXL_RAS_UNC_ERR_STATUS:
528 {
529 uint32_t capctrl = ldl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL);
530 uint32_t fe = FIELD_EX32(capctrl, CXL_RAS_ERR_CAP_CTRL,
531 FIRST_ERROR_POINTER);
532 CXLError *cxl_err;
533 uint32_t unc_err;
534
535 /*
536 * If single bit written that corresponds to the first error
537 * pointer being cleared, update the status and header log.
538 */
539 if (!QTAILQ_EMPTY(&ct3d->error_list)) {
540 if ((1 << fe) ^ value) {
541 CXLError *cxl_next;
542 /*
543 * Software is using wrong flow for multiple header recording
544 * Following behavior in PCIe r6.0 and assuming multiple
545 * header support. Implementation defined choice to clear all
546 * matching records if more than one bit set - which corresponds
547 * closest to behavior of hardware not capable of multiple
548 * header recording.
549 */
550 QTAILQ_FOREACH_SAFE(cxl_err, &ct3d->error_list, node,
551 cxl_next) {
552 if ((1 << cxl_err->type) & value) {
553 QTAILQ_REMOVE(&ct3d->error_list, cxl_err, node);
554 g_free(cxl_err);
555 }
556 }
557 } else {
558 /* Done with previous FE, so drop from list */
559 cxl_err = QTAILQ_FIRST(&ct3d->error_list);
560 QTAILQ_REMOVE(&ct3d->error_list, cxl_err, node);
561 g_free(cxl_err);
562 }
563
564 /*
565 * If there is another FE, then put that in place and update
566 * the header log
567 */
568 if (!QTAILQ_EMPTY(&ct3d->error_list)) {
569 uint32_t *header_log = &cache_mem[R_CXL_RAS_ERR_HEADER0];
570 int i;
571
572 cxl_err = QTAILQ_FIRST(&ct3d->error_list);
573 for (i = 0; i < CXL_RAS_ERR_HEADER_NUM; i++) {
574 stl_le_p(header_log + i, cxl_err->header[i]);
575 }
576 capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL,
577 FIRST_ERROR_POINTER, cxl_err->type);
578 } else {
579 /*
580 * If no more errors, then follow recommendation of PCI spec
581 * r6.0 6.2.4.2 to set the first error pointer to a status
582 * bit that will never be used.
583 */
584 capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL,
585 FIRST_ERROR_POINTER,
586 CXL_RAS_UNC_ERR_CXL_UNUSED);
587 }
588 stl_le_p((uint8_t *)cache_mem + A_CXL_RAS_ERR_CAP_CTRL, capctrl);
589 }
590 unc_err = 0;
591 QTAILQ_FOREACH(cxl_err, &ct3d->error_list, node) {
592 unc_err |= 1 << cxl_err->type;
593 }
594 stl_le_p((uint8_t *)cache_mem + offset, unc_err);
595
596 return;
597 }
598 case A_CXL_RAS_COR_ERR_STATUS:
599 {
600 uint32_t rw1c = value;
601 uint32_t temp = ldl_le_p((uint8_t *)cache_mem + offset);
602 temp &= ~rw1c;
603 stl_le_p((uint8_t *)cache_mem + offset, temp);
604 return;
605 }
606 default:
607 break;
608 }
609
610 stl_le_p((uint8_t *)cache_mem + offset, value);
611 if (should_commit) {
612 hdm_decoder_commit(ct3d, which_hdm);
613 } else if (should_uncommit) {
614 hdm_decoder_uncommit(ct3d, which_hdm);
615 }
616 }
617
618 /*
619 * TODO: dc region configuration will be updated once host backend and address
620 * space support is added for DCD.
621 */
cxl_create_dc_regions(CXLType3Dev * ct3d,Error ** errp)622 static bool cxl_create_dc_regions(CXLType3Dev *ct3d, Error **errp)
623 {
624 int i;
625 uint64_t region_base = 0;
626 uint64_t region_len;
627 uint64_t decode_len;
628 uint64_t blk_size = 2 * MiB;
629 CXLDCRegion *region;
630 MemoryRegion *mr;
631 uint64_t dc_size;
632
633 mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
634 dc_size = memory_region_size(mr);
635 region_len = DIV_ROUND_UP(dc_size, ct3d->dc.num_regions);
636
637 if (dc_size % (ct3d->dc.num_regions * CXL_CAPACITY_MULTIPLIER) != 0) {
638 error_setg(errp,
639 "backend size is not multiple of region len: 0x%" PRIx64,
640 region_len);
641 return false;
642 }
643 if (region_len % CXL_CAPACITY_MULTIPLIER != 0) {
644 error_setg(errp, "DC region size is unaligned to 0x%" PRIx64,
645 CXL_CAPACITY_MULTIPLIER);
646 return false;
647 }
648 decode_len = region_len;
649
650 if (ct3d->hostvmem) {
651 mr = host_memory_backend_get_memory(ct3d->hostvmem);
652 region_base += memory_region_size(mr);
653 }
654 if (ct3d->hostpmem) {
655 mr = host_memory_backend_get_memory(ct3d->hostpmem);
656 region_base += memory_region_size(mr);
657 }
658 if (region_base % CXL_CAPACITY_MULTIPLIER != 0) {
659 error_setg(errp, "DC region base not aligned to 0x%" PRIx64,
660 CXL_CAPACITY_MULTIPLIER);
661 return false;
662 }
663
664 for (i = 0, region = &ct3d->dc.regions[0];
665 i < ct3d->dc.num_regions;
666 i++, region++, region_base += region_len) {
667 *region = (CXLDCRegion) {
668 .base = region_base,
669 .decode_len = decode_len,
670 .len = region_len,
671 .block_size = blk_size,
672 /* dsmad_handle set when creating CDAT table entries */
673 .flags = 0,
674 };
675 ct3d->dc.total_capacity += region->len;
676 region->blk_bitmap = bitmap_new(region->len / region->block_size);
677 }
678 QTAILQ_INIT(&ct3d->dc.extents);
679 QTAILQ_INIT(&ct3d->dc.extents_pending);
680
681 return true;
682 }
683
cxl_destroy_dc_regions(CXLType3Dev * ct3d)684 static void cxl_destroy_dc_regions(CXLType3Dev *ct3d)
685 {
686 CXLDCExtent *ent, *ent_next;
687 CXLDCExtentGroup *group, *group_next;
688 int i;
689 CXLDCRegion *region;
690
691 QTAILQ_FOREACH_SAFE(ent, &ct3d->dc.extents, node, ent_next) {
692 cxl_remove_extent_from_extent_list(&ct3d->dc.extents, ent);
693 }
694
695 QTAILQ_FOREACH_SAFE(group, &ct3d->dc.extents_pending, node, group_next) {
696 QTAILQ_REMOVE(&ct3d->dc.extents_pending, group, node);
697 QTAILQ_FOREACH_SAFE(ent, &group->list, node, ent_next) {
698 cxl_remove_extent_from_extent_list(&group->list, ent);
699 }
700 g_free(group);
701 }
702
703 for (i = 0; i < ct3d->dc.num_regions; i++) {
704 region = &ct3d->dc.regions[i];
705 g_free(region->blk_bitmap);
706 }
707 }
708
cxl_setup_memory(CXLType3Dev * ct3d,Error ** errp)709 static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp)
710 {
711 DeviceState *ds = DEVICE(ct3d);
712
713 if (!ct3d->hostmem && !ct3d->hostvmem && !ct3d->hostpmem
714 && !ct3d->dc.num_regions) {
715 error_setg(errp, "at least one memdev property must be set");
716 return false;
717 } else if (ct3d->hostmem && ct3d->hostpmem) {
718 error_setg(errp, "[memdev] cannot be used with new "
719 "[persistent-memdev] property");
720 return false;
721 } else if (ct3d->hostmem) {
722 /* Use of hostmem property implies pmem */
723 ct3d->hostpmem = ct3d->hostmem;
724 ct3d->hostmem = NULL;
725 }
726
727 if (ct3d->hostpmem && !ct3d->lsa) {
728 error_setg(errp, "lsa property must be set for persistent devices");
729 return false;
730 }
731
732 if (ct3d->hostvmem) {
733 MemoryRegion *vmr;
734 char *v_name;
735
736 vmr = host_memory_backend_get_memory(ct3d->hostvmem);
737 if (!vmr) {
738 error_setg(errp, "volatile memdev must have backing device");
739 return false;
740 }
741 if (host_memory_backend_is_mapped(ct3d->hostvmem)) {
742 error_setg(errp, "memory backend %s can't be used multiple times.",
743 object_get_canonical_path_component(OBJECT(ct3d->hostvmem)));
744 return false;
745 }
746 memory_region_set_nonvolatile(vmr, false);
747 memory_region_set_enabled(vmr, true);
748 host_memory_backend_set_mapped(ct3d->hostvmem, true);
749 if (ds->id) {
750 v_name = g_strdup_printf("cxl-type3-dpa-vmem-space:%s", ds->id);
751 } else {
752 v_name = g_strdup("cxl-type3-dpa-vmem-space");
753 }
754 address_space_init(&ct3d->hostvmem_as, vmr, v_name);
755 ct3d->cxl_dstate.vmem_size = memory_region_size(vmr);
756 ct3d->cxl_dstate.static_mem_size += memory_region_size(vmr);
757 g_free(v_name);
758 }
759
760 if (ct3d->hostpmem) {
761 MemoryRegion *pmr;
762 char *p_name;
763
764 pmr = host_memory_backend_get_memory(ct3d->hostpmem);
765 if (!pmr) {
766 error_setg(errp, "persistent memdev must have backing device");
767 return false;
768 }
769 if (host_memory_backend_is_mapped(ct3d->hostpmem)) {
770 error_setg(errp, "memory backend %s can't be used multiple times.",
771 object_get_canonical_path_component(OBJECT(ct3d->hostpmem)));
772 return false;
773 }
774 memory_region_set_nonvolatile(pmr, true);
775 memory_region_set_enabled(pmr, true);
776 host_memory_backend_set_mapped(ct3d->hostpmem, true);
777 if (ds->id) {
778 p_name = g_strdup_printf("cxl-type3-dpa-pmem-space:%s", ds->id);
779 } else {
780 p_name = g_strdup("cxl-type3-dpa-pmem-space");
781 }
782 address_space_init(&ct3d->hostpmem_as, pmr, p_name);
783 ct3d->cxl_dstate.pmem_size = memory_region_size(pmr);
784 ct3d->cxl_dstate.static_mem_size += memory_region_size(pmr);
785 g_free(p_name);
786 }
787
788 ct3d->dc.total_capacity = 0;
789 if (ct3d->dc.num_regions > 0) {
790 MemoryRegion *dc_mr;
791 char *dc_name;
792
793 if (!ct3d->dc.host_dc) {
794 error_setg(errp, "dynamic capacity must have a backing device");
795 return false;
796 }
797
798 dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
799 if (!dc_mr) {
800 error_setg(errp, "dynamic capacity must have a backing device");
801 return false;
802 }
803
804 if (host_memory_backend_is_mapped(ct3d->dc.host_dc)) {
805 error_setg(errp, "memory backend %s can't be used multiple times.",
806 object_get_canonical_path_component(OBJECT(ct3d->dc.host_dc)));
807 return false;
808 }
809 /*
810 * Set DC regions as volatile for now, non-volatile support can
811 * be added in the future if needed.
812 */
813 memory_region_set_nonvolatile(dc_mr, false);
814 memory_region_set_enabled(dc_mr, true);
815 host_memory_backend_set_mapped(ct3d->dc.host_dc, true);
816 if (ds->id) {
817 dc_name = g_strdup_printf("cxl-dcd-dpa-dc-space:%s", ds->id);
818 } else {
819 dc_name = g_strdup("cxl-dcd-dpa-dc-space");
820 }
821 address_space_init(&ct3d->dc.host_dc_as, dc_mr, dc_name);
822 g_free(dc_name);
823
824 if (!cxl_create_dc_regions(ct3d, errp)) {
825 error_append_hint(errp, "setup DC regions failed");
826 return false;
827 }
828 }
829
830 return true;
831 }
832
833 static DOEProtocol doe_cdat_prot[] = {
834 { CXL_VENDOR_ID, CXL_DOE_TABLE_ACCESS, cxl_doe_cdat_rsp },
835 { }
836 };
837
ct3_realize(PCIDevice * pci_dev,Error ** errp)838 static void ct3_realize(PCIDevice *pci_dev, Error **errp)
839 {
840 ERRP_GUARD();
841 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev);
842 CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;
843 ComponentRegisters *regs = &cxl_cstate->crb;
844 MemoryRegion *mr = ®s->component_registers;
845 uint8_t *pci_conf = pci_dev->config;
846 unsigned short msix_num = 6;
847 int i, rc;
848 uint16_t count;
849
850 QTAILQ_INIT(&ct3d->error_list);
851
852 if (!cxl_setup_memory(ct3d, errp)) {
853 return;
854 }
855
856 pci_config_set_prog_interface(pci_conf, 0x10);
857
858 pcie_endpoint_cap_init(pci_dev, 0x80);
859 if (ct3d->sn != UI64_NULL) {
860 pcie_dev_ser_num_init(pci_dev, 0x100, ct3d->sn);
861 cxl_cstate->dvsec_offset = 0x100 + 0x0c;
862 } else {
863 cxl_cstate->dvsec_offset = 0x100;
864 }
865
866 ct3d->cxl_cstate.pdev = pci_dev;
867 build_dvsecs(ct3d);
868
869 regs->special_ops = g_new0(MemoryRegionOps, 1);
870 regs->special_ops->write = ct3d_reg_write;
871
872 cxl_component_register_block_init(OBJECT(pci_dev), cxl_cstate,
873 TYPE_CXL_TYPE3);
874
875 pci_register_bar(
876 pci_dev, CXL_COMPONENT_REG_BAR_IDX,
877 PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64, mr);
878
879 cxl_device_register_block_init(OBJECT(pci_dev), &ct3d->cxl_dstate,
880 &ct3d->cci);
881 pci_register_bar(pci_dev, CXL_DEVICE_REG_BAR_IDX,
882 PCI_BASE_ADDRESS_SPACE_MEMORY |
883 PCI_BASE_ADDRESS_MEM_TYPE_64,
884 &ct3d->cxl_dstate.device_registers);
885
886 /* MSI(-X) Initialization */
887 rc = msix_init_exclusive_bar(pci_dev, msix_num, 4, NULL);
888 if (rc) {
889 goto err_address_space_free;
890 }
891 for (i = 0; i < msix_num; i++) {
892 msix_vector_use(pci_dev, i);
893 }
894
895 /* DOE Initialization */
896 pcie_doe_init(pci_dev, &ct3d->doe_cdat, 0x190, doe_cdat_prot, true, 0);
897
898 cxl_cstate->cdat.build_cdat_table = ct3_build_cdat_table;
899 cxl_cstate->cdat.free_cdat_table = ct3_free_cdat_table;
900 cxl_cstate->cdat.private = ct3d;
901 if (!cxl_doe_cdat_init(cxl_cstate, errp)) {
902 goto err_free_special_ops;
903 }
904
905 pcie_cap_deverr_init(pci_dev);
906 /* Leave a bit of room for expansion */
907 rc = pcie_aer_init(pci_dev, PCI_ERR_VER, 0x200, PCI_ERR_SIZEOF, NULL);
908 if (rc) {
909 goto err_release_cdat;
910 }
911 cxl_event_init(&ct3d->cxl_dstate, 2);
912
913 /* Set default value for patrol scrub attributes */
914 ct3d->patrol_scrub_attrs.scrub_cycle_cap =
915 CXL_MEMDEV_PS_SCRUB_CYCLE_CHANGE_CAP_DEFAULT |
916 CXL_MEMDEV_PS_SCRUB_REALTIME_REPORT_CAP_DEFAULT;
917 ct3d->patrol_scrub_attrs.scrub_cycle =
918 CXL_MEMDEV_PS_CUR_SCRUB_CYCLE_DEFAULT |
919 (CXL_MEMDEV_PS_MIN_SCRUB_CYCLE_DEFAULT << 8);
920 ct3d->patrol_scrub_attrs.scrub_flags = CXL_MEMDEV_PS_ENABLE_DEFAULT;
921
922 /* Set default value for DDR5 ECS read attributes */
923 ct3d->ecs_attrs.ecs_log_cap = CXL_ECS_LOG_ENTRY_TYPE_DEFAULT;
924 for (count = 0; count < CXL_ECS_NUM_MEDIA_FRUS; count++) {
925 ct3d->ecs_attrs.fru_attrs[count].ecs_cap =
926 CXL_ECS_REALTIME_REPORT_CAP_DEFAULT;
927 ct3d->ecs_attrs.fru_attrs[count].ecs_config =
928 CXL_ECS_THRESHOLD_COUNT_DEFAULT |
929 (CXL_ECS_MODE_DEFAULT << 3);
930 /* Reserved */
931 ct3d->ecs_attrs.fru_attrs[count].ecs_flags = 0;
932 }
933
934 return;
935
936 err_release_cdat:
937 cxl_doe_cdat_release(cxl_cstate);
938 err_free_special_ops:
939 g_free(regs->special_ops);
940 err_address_space_free:
941 if (ct3d->dc.host_dc) {
942 cxl_destroy_dc_regions(ct3d);
943 address_space_destroy(&ct3d->dc.host_dc_as);
944 }
945 if (ct3d->hostpmem) {
946 address_space_destroy(&ct3d->hostpmem_as);
947 }
948 if (ct3d->hostvmem) {
949 address_space_destroy(&ct3d->hostvmem_as);
950 }
951 return;
952 }
953
ct3_exit(PCIDevice * pci_dev)954 static void ct3_exit(PCIDevice *pci_dev)
955 {
956 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev);
957 CXLComponentState *cxl_cstate = &ct3d->cxl_cstate;
958 ComponentRegisters *regs = &cxl_cstate->crb;
959
960 pcie_aer_exit(pci_dev);
961 cxl_doe_cdat_release(cxl_cstate);
962 g_free(regs->special_ops);
963 if (ct3d->dc.host_dc) {
964 cxl_destroy_dc_regions(ct3d);
965 address_space_destroy(&ct3d->dc.host_dc_as);
966 }
967 if (ct3d->hostpmem) {
968 address_space_destroy(&ct3d->hostpmem_as);
969 }
970 if (ct3d->hostvmem) {
971 address_space_destroy(&ct3d->hostvmem_as);
972 }
973 }
974
975 /*
976 * Mark the DPA range [dpa, dap + len - 1] to be backed and accessible. This
977 * happens when a DC extent is added and accepted by the host.
978 */
ct3_set_region_block_backed(CXLType3Dev * ct3d,uint64_t dpa,uint64_t len)979 void ct3_set_region_block_backed(CXLType3Dev *ct3d, uint64_t dpa,
980 uint64_t len)
981 {
982 CXLDCRegion *region;
983
984 region = cxl_find_dc_region(ct3d, dpa, len);
985 if (!region) {
986 return;
987 }
988
989 bitmap_set(region->blk_bitmap, (dpa - region->base) / region->block_size,
990 len / region->block_size);
991 }
992
993 /*
994 * Check whether the DPA range [dpa, dpa + len - 1] is backed with DC extents.
995 * Used when validating read/write to dc regions
996 */
ct3_test_region_block_backed(CXLType3Dev * ct3d,uint64_t dpa,uint64_t len)997 bool ct3_test_region_block_backed(CXLType3Dev *ct3d, uint64_t dpa,
998 uint64_t len)
999 {
1000 CXLDCRegion *region;
1001 uint64_t nbits;
1002 long nr;
1003
1004 region = cxl_find_dc_region(ct3d, dpa, len);
1005 if (!region) {
1006 return false;
1007 }
1008
1009 nr = (dpa - region->base) / region->block_size;
1010 nbits = DIV_ROUND_UP(len, region->block_size);
1011 /*
1012 * if bits between [dpa, dpa + len) are all 1s, meaning the DPA range is
1013 * backed with DC extents, return true; else return false.
1014 */
1015 return find_next_zero_bit(region->blk_bitmap, nr + nbits, nr) == nr + nbits;
1016 }
1017
1018 /*
1019 * Mark the DPA range [dpa, dap + len - 1] to be unbacked and inaccessible.
1020 * This happens when a dc extent is released by the host.
1021 */
ct3_clear_region_block_backed(CXLType3Dev * ct3d,uint64_t dpa,uint64_t len)1022 void ct3_clear_region_block_backed(CXLType3Dev *ct3d, uint64_t dpa,
1023 uint64_t len)
1024 {
1025 CXLDCRegion *region;
1026 uint64_t nbits;
1027 long nr;
1028
1029 region = cxl_find_dc_region(ct3d, dpa, len);
1030 if (!region) {
1031 return;
1032 }
1033
1034 nr = (dpa - region->base) / region->block_size;
1035 nbits = len / region->block_size;
1036 bitmap_clear(region->blk_bitmap, nr, nbits);
1037 }
1038
cxl_type3_dpa(CXLType3Dev * ct3d,hwaddr host_addr,uint64_t * dpa)1039 static bool cxl_type3_dpa(CXLType3Dev *ct3d, hwaddr host_addr, uint64_t *dpa)
1040 {
1041 int hdm_inc = R_CXL_HDM_DECODER1_BASE_LO - R_CXL_HDM_DECODER0_BASE_LO;
1042 uint32_t *cache_mem = ct3d->cxl_cstate.crb.cache_mem_registers;
1043 unsigned int hdm_count;
1044 uint32_t cap;
1045 uint64_t dpa_base = 0;
1046 int i;
1047
1048 cap = ldl_le_p(cache_mem + R_CXL_HDM_DECODER_CAPABILITY);
1049 hdm_count = cxl_decoder_count_dec(FIELD_EX32(cap,
1050 CXL_HDM_DECODER_CAPABILITY,
1051 DECODER_COUNT));
1052
1053 for (i = 0; i < hdm_count; i++) {
1054 uint64_t decoder_base, decoder_size, hpa_offset, skip;
1055 uint32_t hdm_ctrl, low, high;
1056 int ig, iw;
1057
1058 low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_LO + i * hdm_inc);
1059 high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_BASE_HI + i * hdm_inc);
1060 decoder_base = ((uint64_t)high << 32) | (low & 0xf0000000);
1061
1062 low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_LO + i * hdm_inc);
1063 high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_SIZE_HI + i * hdm_inc);
1064 decoder_size = ((uint64_t)high << 32) | (low & 0xf0000000);
1065
1066 low = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_DPA_SKIP_LO +
1067 i * hdm_inc);
1068 high = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_DPA_SKIP_HI +
1069 i * hdm_inc);
1070 skip = ((uint64_t)high << 32) | (low & 0xf0000000);
1071 dpa_base += skip;
1072
1073 hpa_offset = (uint64_t)host_addr - decoder_base;
1074
1075 hdm_ctrl = ldl_le_p(cache_mem + R_CXL_HDM_DECODER0_CTRL + i * hdm_inc);
1076 iw = FIELD_EX32(hdm_ctrl, CXL_HDM_DECODER0_CTRL, IW);
1077 ig = FIELD_EX32(hdm_ctrl, CXL_HDM_DECODER0_CTRL, IG);
1078 if (!FIELD_EX32(hdm_ctrl, CXL_HDM_DECODER0_CTRL, COMMITTED)) {
1079 return false;
1080 }
1081 if (((uint64_t)host_addr < decoder_base) ||
1082 (hpa_offset >= decoder_size)) {
1083 int decoded_iw = cxl_interleave_ways_dec(iw, &error_fatal);
1084
1085 if (decoded_iw == 0) {
1086 return false;
1087 }
1088
1089 dpa_base += decoder_size / decoded_iw;
1090 continue;
1091 }
1092
1093 *dpa = dpa_base +
1094 ((MAKE_64BIT_MASK(0, 8 + ig) & hpa_offset) |
1095 ((MAKE_64BIT_MASK(8 + ig + iw, 64 - 8 - ig - iw) & hpa_offset)
1096 >> iw));
1097
1098 return true;
1099 }
1100 return false;
1101 }
1102
cxl_type3_hpa_to_as_and_dpa(CXLType3Dev * ct3d,hwaddr host_addr,unsigned int size,AddressSpace ** as,uint64_t * dpa_offset)1103 static int cxl_type3_hpa_to_as_and_dpa(CXLType3Dev *ct3d,
1104 hwaddr host_addr,
1105 unsigned int size,
1106 AddressSpace **as,
1107 uint64_t *dpa_offset)
1108 {
1109 MemoryRegion *vmr = NULL, *pmr = NULL, *dc_mr = NULL;
1110 uint64_t vmr_size = 0, pmr_size = 0, dc_size = 0;
1111
1112 if (ct3d->hostvmem) {
1113 vmr = host_memory_backend_get_memory(ct3d->hostvmem);
1114 vmr_size = memory_region_size(vmr);
1115 }
1116 if (ct3d->hostpmem) {
1117 pmr = host_memory_backend_get_memory(ct3d->hostpmem);
1118 pmr_size = memory_region_size(pmr);
1119 }
1120 if (ct3d->dc.host_dc) {
1121 dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
1122 dc_size = memory_region_size(dc_mr);
1123 }
1124
1125 if (!vmr && !pmr && !dc_mr) {
1126 return -ENODEV;
1127 }
1128
1129 if (!cxl_type3_dpa(ct3d, host_addr, dpa_offset)) {
1130 return -EINVAL;
1131 }
1132
1133 if (*dpa_offset >= vmr_size + pmr_size + dc_size) {
1134 return -EINVAL;
1135 }
1136
1137 if (*dpa_offset < vmr_size) {
1138 *as = &ct3d->hostvmem_as;
1139 } else if (*dpa_offset < vmr_size + pmr_size) {
1140 *as = &ct3d->hostpmem_as;
1141 *dpa_offset -= vmr_size;
1142 } else {
1143 if (!ct3_test_region_block_backed(ct3d, *dpa_offset, size)) {
1144 return -ENODEV;
1145 }
1146
1147 *as = &ct3d->dc.host_dc_as;
1148 *dpa_offset -= (vmr_size + pmr_size);
1149 }
1150
1151 return 0;
1152 }
1153
cxl_type3_read(PCIDevice * d,hwaddr host_addr,uint64_t * data,unsigned size,MemTxAttrs attrs)1154 MemTxResult cxl_type3_read(PCIDevice *d, hwaddr host_addr, uint64_t *data,
1155 unsigned size, MemTxAttrs attrs)
1156 {
1157 CXLType3Dev *ct3d = CXL_TYPE3(d);
1158 uint64_t dpa_offset = 0;
1159 AddressSpace *as = NULL;
1160 int res;
1161
1162 res = cxl_type3_hpa_to_as_and_dpa(ct3d, host_addr, size,
1163 &as, &dpa_offset);
1164 if (res) {
1165 return MEMTX_ERROR;
1166 }
1167
1168 if (cxl_dev_media_disabled(&ct3d->cxl_dstate)) {
1169 qemu_guest_getrandom_nofail(data, size);
1170 return MEMTX_OK;
1171 }
1172
1173 return address_space_read(as, dpa_offset, attrs, data, size);
1174 }
1175
cxl_type3_write(PCIDevice * d,hwaddr host_addr,uint64_t data,unsigned size,MemTxAttrs attrs)1176 MemTxResult cxl_type3_write(PCIDevice *d, hwaddr host_addr, uint64_t data,
1177 unsigned size, MemTxAttrs attrs)
1178 {
1179 CXLType3Dev *ct3d = CXL_TYPE3(d);
1180 uint64_t dpa_offset = 0;
1181 AddressSpace *as = NULL;
1182 int res;
1183
1184 res = cxl_type3_hpa_to_as_and_dpa(ct3d, host_addr, size,
1185 &as, &dpa_offset);
1186 if (res) {
1187 return MEMTX_ERROR;
1188 }
1189
1190 if (cxl_dev_media_disabled(&ct3d->cxl_dstate)) {
1191 return MEMTX_OK;
1192 }
1193
1194 return address_space_write(as, dpa_offset, attrs, &data, size);
1195 }
1196
ct3d_reset(DeviceState * dev)1197 static void ct3d_reset(DeviceState *dev)
1198 {
1199 CXLType3Dev *ct3d = CXL_TYPE3(dev);
1200 uint32_t *reg_state = ct3d->cxl_cstate.crb.cache_mem_registers;
1201 uint32_t *write_msk = ct3d->cxl_cstate.crb.cache_mem_regs_write_mask;
1202
1203 pcie_cap_fill_link_ep_usp(PCI_DEVICE(dev), ct3d->width, ct3d->speed);
1204 cxl_component_register_init_common(reg_state, write_msk, CXL2_TYPE3_DEVICE);
1205 cxl_device_register_init_t3(ct3d);
1206
1207 /*
1208 * Bring up an endpoint to target with MCTP over VDM.
1209 * This device is emulating an MLD with single LD for now.
1210 */
1211 cxl_initialize_t3_fm_owned_ld_mctpcci(&ct3d->vdm_fm_owned_ld_mctp_cci,
1212 DEVICE(ct3d), DEVICE(ct3d),
1213 512); /* Max payload made up */
1214 cxl_initialize_t3_ld_cci(&ct3d->ld0_cci, DEVICE(ct3d), DEVICE(ct3d),
1215 512); /* Max payload made up */
1216
1217 }
1218
1219 static Property ct3_props[] = {
1220 DEFINE_PROP_LINK("memdev", CXLType3Dev, hostmem, TYPE_MEMORY_BACKEND,
1221 HostMemoryBackend *), /* for backward compatibility */
1222 DEFINE_PROP_LINK("persistent-memdev", CXLType3Dev, hostpmem,
1223 TYPE_MEMORY_BACKEND, HostMemoryBackend *),
1224 DEFINE_PROP_LINK("volatile-memdev", CXLType3Dev, hostvmem,
1225 TYPE_MEMORY_BACKEND, HostMemoryBackend *),
1226 DEFINE_PROP_LINK("lsa", CXLType3Dev, lsa, TYPE_MEMORY_BACKEND,
1227 HostMemoryBackend *),
1228 DEFINE_PROP_UINT64("sn", CXLType3Dev, sn, UI64_NULL),
1229 DEFINE_PROP_STRING("cdat", CXLType3Dev, cxl_cstate.cdat.filename),
1230 DEFINE_PROP_UINT8("num-dc-regions", CXLType3Dev, dc.num_regions, 0),
1231 DEFINE_PROP_LINK("volatile-dc-memdev", CXLType3Dev, dc.host_dc,
1232 TYPE_MEMORY_BACKEND, HostMemoryBackend *),
1233 DEFINE_PROP_PCIE_LINK_SPEED("x-speed", CXLType3Dev,
1234 speed, PCIE_LINK_SPEED_32),
1235 DEFINE_PROP_PCIE_LINK_WIDTH("x-width", CXLType3Dev,
1236 width, PCIE_LINK_WIDTH_16),
1237 DEFINE_PROP_END_OF_LIST(),
1238 };
1239
get_lsa_size(CXLType3Dev * ct3d)1240 static uint64_t get_lsa_size(CXLType3Dev *ct3d)
1241 {
1242 MemoryRegion *mr;
1243
1244 if (!ct3d->lsa) {
1245 return 0;
1246 }
1247
1248 mr = host_memory_backend_get_memory(ct3d->lsa);
1249 return memory_region_size(mr);
1250 }
1251
validate_lsa_access(MemoryRegion * mr,uint64_t size,uint64_t offset)1252 static void validate_lsa_access(MemoryRegion *mr, uint64_t size,
1253 uint64_t offset)
1254 {
1255 assert(offset + size <= memory_region_size(mr));
1256 assert(offset + size > offset);
1257 }
1258
get_lsa(CXLType3Dev * ct3d,void * buf,uint64_t size,uint64_t offset)1259 static uint64_t get_lsa(CXLType3Dev *ct3d, void *buf, uint64_t size,
1260 uint64_t offset)
1261 {
1262 MemoryRegion *mr;
1263 void *lsa;
1264
1265 if (!ct3d->lsa) {
1266 return 0;
1267 }
1268
1269 mr = host_memory_backend_get_memory(ct3d->lsa);
1270 validate_lsa_access(mr, size, offset);
1271
1272 lsa = memory_region_get_ram_ptr(mr) + offset;
1273 memcpy(buf, lsa, size);
1274
1275 return size;
1276 }
1277
set_lsa(CXLType3Dev * ct3d,const void * buf,uint64_t size,uint64_t offset)1278 static void set_lsa(CXLType3Dev *ct3d, const void *buf, uint64_t size,
1279 uint64_t offset)
1280 {
1281 MemoryRegion *mr;
1282 void *lsa;
1283
1284 if (!ct3d->lsa) {
1285 return;
1286 }
1287
1288 mr = host_memory_backend_get_memory(ct3d->lsa);
1289 validate_lsa_access(mr, size, offset);
1290
1291 lsa = memory_region_get_ram_ptr(mr) + offset;
1292 memcpy(lsa, buf, size);
1293 memory_region_set_dirty(mr, offset, size);
1294
1295 /*
1296 * Just like the PMEM, if the guest is not allowed to exit gracefully, label
1297 * updates will get lost.
1298 */
1299 }
1300
set_cacheline(CXLType3Dev * ct3d,uint64_t dpa_offset,uint8_t * data)1301 static bool set_cacheline(CXLType3Dev *ct3d, uint64_t dpa_offset, uint8_t *data)
1302 {
1303 MemoryRegion *vmr = NULL, *pmr = NULL, *dc_mr = NULL;
1304 AddressSpace *as;
1305 uint64_t vmr_size = 0, pmr_size = 0, dc_size = 0;
1306
1307 if (ct3d->hostvmem) {
1308 vmr = host_memory_backend_get_memory(ct3d->hostvmem);
1309 vmr_size = memory_region_size(vmr);
1310 }
1311 if (ct3d->hostpmem) {
1312 pmr = host_memory_backend_get_memory(ct3d->hostpmem);
1313 pmr_size = memory_region_size(pmr);
1314 }
1315 if (ct3d->dc.host_dc) {
1316 dc_mr = host_memory_backend_get_memory(ct3d->dc.host_dc);
1317 dc_size = memory_region_size(dc_mr);
1318 }
1319
1320 if (!vmr && !pmr && !dc_mr) {
1321 return false;
1322 }
1323
1324 if (dpa_offset + CXL_CACHE_LINE_SIZE > vmr_size + pmr_size + dc_size) {
1325 return false;
1326 }
1327
1328 if (dpa_offset < vmr_size) {
1329 as = &ct3d->hostvmem_as;
1330 } else if (dpa_offset < vmr_size + pmr_size) {
1331 as = &ct3d->hostpmem_as;
1332 dpa_offset -= vmr_size;
1333 } else {
1334 as = &ct3d->dc.host_dc_as;
1335 dpa_offset -= (vmr_size + pmr_size);
1336 }
1337
1338 address_space_write(as, dpa_offset, MEMTXATTRS_UNSPECIFIED, data,
1339 CXL_CACHE_LINE_SIZE);
1340 return true;
1341 }
1342
cxl_set_poison_list_overflowed(CXLType3Dev * ct3d)1343 void cxl_set_poison_list_overflowed(CXLType3Dev *ct3d)
1344 {
1345 ct3d->poison_list_overflowed = true;
1346 ct3d->poison_list_overflow_ts =
1347 cxl_device_get_timestamp(&ct3d->cxl_dstate);
1348 }
1349
cxl_clear_poison_list_overflowed(CXLType3Dev * ct3d)1350 void cxl_clear_poison_list_overflowed(CXLType3Dev *ct3d)
1351 {
1352 ct3d->poison_list_overflowed = false;
1353 ct3d->poison_list_overflow_ts = 0;
1354 }
1355
qmp_cxl_inject_poison(const char * path,uint64_t start,uint64_t length,Error ** errp)1356 void qmp_cxl_inject_poison(const char *path, uint64_t start, uint64_t length,
1357 Error **errp)
1358 {
1359 Object *obj = object_resolve_path(path, NULL);
1360 CXLType3Dev *ct3d;
1361 CXLPoison *p;
1362
1363 if (length % 64) {
1364 error_setg(errp, "Poison injection must be in multiples of 64 bytes");
1365 return;
1366 }
1367 if (start % 64) {
1368 error_setg(errp, "Poison start address must be 64 byte aligned");
1369 return;
1370 }
1371 if (!obj) {
1372 error_setg(errp, "Unable to resolve path");
1373 return;
1374 }
1375 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1376 error_setg(errp, "Path does not point to a CXL type 3 device");
1377 return;
1378 }
1379
1380 ct3d = CXL_TYPE3(obj);
1381
1382 QLIST_FOREACH(p, &ct3d->poison_list, node) {
1383 if ((start < p->start + p->length) && (start + length > p->start)) {
1384 error_setg(errp,
1385 "Overlap with existing poisoned region not supported");
1386 return;
1387 }
1388 }
1389
1390 p = g_new0(CXLPoison, 1);
1391 p->length = length;
1392 p->start = start;
1393 /* Different from injected via the mbox */
1394 p->type = CXL_POISON_TYPE_INTERNAL;
1395
1396 if (ct3d->poison_list_cnt < CXL_POISON_LIST_LIMIT) {
1397 QLIST_INSERT_HEAD(&ct3d->poison_list, p, node);
1398 ct3d->poison_list_cnt++;
1399 } else {
1400 if (!ct3d->poison_list_overflowed) {
1401 cxl_set_poison_list_overflowed(ct3d);
1402 }
1403 QLIST_INSERT_HEAD(&ct3d->poison_list_bkp, p, node);
1404 }
1405 }
1406
1407 /* For uncorrectable errors include support for multiple header recording */
qmp_cxl_inject_uncorrectable_errors(const char * path,CXLUncorErrorRecordList * errors,Error ** errp)1408 void qmp_cxl_inject_uncorrectable_errors(const char *path,
1409 CXLUncorErrorRecordList *errors,
1410 Error **errp)
1411 {
1412 Object *obj = object_resolve_path(path, NULL);
1413 static PCIEAERErr err = {};
1414 CXLType3Dev *ct3d;
1415 CXLError *cxl_err;
1416 uint32_t *reg_state;
1417 uint32_t unc_err;
1418 bool first;
1419
1420 if (!obj) {
1421 error_setg(errp, "Unable to resolve path");
1422 return;
1423 }
1424
1425 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1426 error_setg(errp, "Path does not point to a CXL type 3 device");
1427 return;
1428 }
1429
1430 err.status = PCI_ERR_UNC_INTN;
1431 err.source_id = pci_requester_id(PCI_DEVICE(obj));
1432 err.flags = 0;
1433
1434 ct3d = CXL_TYPE3(obj);
1435
1436 first = QTAILQ_EMPTY(&ct3d->error_list);
1437 reg_state = ct3d->cxl_cstate.crb.cache_mem_registers;
1438 while (errors) {
1439 uint32List *header = errors->value->header;
1440 uint8_t header_count = 0;
1441 int cxl_err_code;
1442
1443 cxl_err_code = ct3d_qmp_uncor_err_to_cxl(errors->value->type);
1444 if (cxl_err_code < 0) {
1445 error_setg(errp, "Unknown error code");
1446 return;
1447 }
1448
1449 /* If the error is masked, nothing to do here */
1450 if (!((1 << cxl_err_code) &
1451 ~ldl_le_p(reg_state + R_CXL_RAS_UNC_ERR_MASK))) {
1452 errors = errors->next;
1453 continue;
1454 }
1455
1456 cxl_err = g_malloc0(sizeof(*cxl_err));
1457
1458 cxl_err->type = cxl_err_code;
1459 while (header && header_count < 32) {
1460 cxl_err->header[header_count++] = header->value;
1461 header = header->next;
1462 }
1463 if (header_count > 32) {
1464 error_setg(errp, "Header must be 32 DWORD or less");
1465 return;
1466 }
1467 QTAILQ_INSERT_TAIL(&ct3d->error_list, cxl_err, node);
1468
1469 errors = errors->next;
1470 }
1471
1472 if (first && !QTAILQ_EMPTY(&ct3d->error_list)) {
1473 uint32_t *cache_mem = ct3d->cxl_cstate.crb.cache_mem_registers;
1474 uint32_t capctrl = ldl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL);
1475 uint32_t *header_log = &cache_mem[R_CXL_RAS_ERR_HEADER0];
1476 int i;
1477
1478 cxl_err = QTAILQ_FIRST(&ct3d->error_list);
1479 for (i = 0; i < CXL_RAS_ERR_HEADER_NUM; i++) {
1480 stl_le_p(header_log + i, cxl_err->header[i]);
1481 }
1482
1483 capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL,
1484 FIRST_ERROR_POINTER, cxl_err->type);
1485 stl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL, capctrl);
1486 }
1487
1488 unc_err = 0;
1489 QTAILQ_FOREACH(cxl_err, &ct3d->error_list, node) {
1490 unc_err |= (1 << cxl_err->type);
1491 }
1492 if (!unc_err) {
1493 return;
1494 }
1495
1496 stl_le_p(reg_state + R_CXL_RAS_UNC_ERR_STATUS, unc_err);
1497 pcie_aer_inject_error(PCI_DEVICE(obj), &err);
1498
1499 return;
1500 }
1501
qmp_cxl_inject_correctable_error(const char * path,CxlCorErrorType type,Error ** errp)1502 void qmp_cxl_inject_correctable_error(const char *path, CxlCorErrorType type,
1503 Error **errp)
1504 {
1505 static PCIEAERErr err = {};
1506 Object *obj = object_resolve_path(path, NULL);
1507 CXLType3Dev *ct3d;
1508 uint32_t *reg_state;
1509 uint32_t cor_err;
1510 int cxl_err_type;
1511
1512 if (!obj) {
1513 error_setg(errp, "Unable to resolve path");
1514 return;
1515 }
1516 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1517 error_setg(errp, "Path does not point to a CXL type 3 device");
1518 return;
1519 }
1520
1521 err.status = PCI_ERR_COR_INTERNAL;
1522 err.source_id = pci_requester_id(PCI_DEVICE(obj));
1523 err.flags = PCIE_AER_ERR_IS_CORRECTABLE;
1524
1525 ct3d = CXL_TYPE3(obj);
1526 reg_state = ct3d->cxl_cstate.crb.cache_mem_registers;
1527 cor_err = ldl_le_p(reg_state + R_CXL_RAS_COR_ERR_STATUS);
1528
1529 cxl_err_type = ct3d_qmp_cor_err_to_cxl(type);
1530 if (cxl_err_type < 0) {
1531 error_setg(errp, "Invalid COR error");
1532 return;
1533 }
1534 /* If the error is masked, nothting to do here */
1535 if (!((1 << cxl_err_type) &
1536 ~ldl_le_p(reg_state + R_CXL_RAS_COR_ERR_MASK))) {
1537 return;
1538 }
1539
1540 cor_err |= (1 << cxl_err_type);
1541 stl_le_p(reg_state + R_CXL_RAS_COR_ERR_STATUS, cor_err);
1542
1543 pcie_aer_inject_error(PCI_DEVICE(obj), &err);
1544 }
1545
cxl_assign_event_header(CXLEventRecordHdr * hdr,const QemuUUID * uuid,uint32_t flags,uint8_t length,uint64_t timestamp)1546 static void cxl_assign_event_header(CXLEventRecordHdr *hdr,
1547 const QemuUUID *uuid, uint32_t flags,
1548 uint8_t length, uint64_t timestamp)
1549 {
1550 st24_le_p(&hdr->flags, flags);
1551 hdr->length = length;
1552 memcpy(&hdr->id, uuid, sizeof(hdr->id));
1553 stq_le_p(&hdr->timestamp, timestamp);
1554 }
1555
1556 static const QemuUUID gen_media_uuid = {
1557 .data = UUID(0xfbcd0a77, 0xc260, 0x417f,
1558 0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6),
1559 };
1560
1561 static const QemuUUID dram_uuid = {
1562 .data = UUID(0x601dcbb3, 0x9c06, 0x4eab, 0xb8, 0xaf,
1563 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24),
1564 };
1565
1566 static const QemuUUID memory_module_uuid = {
1567 .data = UUID(0xfe927475, 0xdd59, 0x4339, 0xa5, 0x86,
1568 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74),
1569 };
1570
1571 #define CXL_GMER_VALID_CHANNEL BIT(0)
1572 #define CXL_GMER_VALID_RANK BIT(1)
1573 #define CXL_GMER_VALID_DEVICE BIT(2)
1574 #define CXL_GMER_VALID_COMPONENT BIT(3)
1575
ct3d_qmp_cxl_event_log_enc(CxlEventLog log)1576 static int ct3d_qmp_cxl_event_log_enc(CxlEventLog log)
1577 {
1578 switch (log) {
1579 case CXL_EVENT_LOG_INFORMATIONAL:
1580 return CXL_EVENT_TYPE_INFO;
1581 case CXL_EVENT_LOG_WARNING:
1582 return CXL_EVENT_TYPE_WARN;
1583 case CXL_EVENT_LOG_FAILURE:
1584 return CXL_EVENT_TYPE_FAIL;
1585 case CXL_EVENT_LOG_FATAL:
1586 return CXL_EVENT_TYPE_FATAL;
1587 default:
1588 return -EINVAL;
1589 }
1590 }
1591 /* Component ID is device specific. Define this as a string. */
qmp_cxl_inject_general_media_event(const char * path,CxlEventLog log,uint8_t flags,uint64_t dpa,uint8_t descriptor,uint8_t type,uint8_t transaction_type,bool has_channel,uint8_t channel,bool has_rank,uint8_t rank,bool has_device,uint32_t device,const char * component_id,Error ** errp)1592 void qmp_cxl_inject_general_media_event(const char *path, CxlEventLog log,
1593 uint8_t flags, uint64_t dpa,
1594 uint8_t descriptor, uint8_t type,
1595 uint8_t transaction_type,
1596 bool has_channel, uint8_t channel,
1597 bool has_rank, uint8_t rank,
1598 bool has_device, uint32_t device,
1599 const char *component_id,
1600 Error **errp)
1601 {
1602 Object *obj = object_resolve_path(path, NULL);
1603 CXLEventGenMedia gem;
1604 CXLEventRecordHdr *hdr = &gem.hdr;
1605 CXLDeviceState *cxlds;
1606 CXLType3Dev *ct3d;
1607 uint16_t valid_flags = 0;
1608 uint8_t enc_log;
1609 int rc;
1610
1611 if (!obj) {
1612 error_setg(errp, "Unable to resolve path");
1613 return;
1614 }
1615 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1616 error_setg(errp, "Path does not point to a CXL type 3 device");
1617 return;
1618 }
1619 ct3d = CXL_TYPE3(obj);
1620 cxlds = &ct3d->cxl_dstate;
1621
1622 rc = ct3d_qmp_cxl_event_log_enc(log);
1623 if (rc < 0) {
1624 error_setg(errp, "Unhandled error log type");
1625 return;
1626 }
1627 enc_log = rc;
1628
1629 memset(&gem, 0, sizeof(gem));
1630 cxl_assign_event_header(hdr, &gen_media_uuid, flags, sizeof(gem),
1631 cxl_device_get_timestamp(&ct3d->cxl_dstate));
1632
1633 stq_le_p(&gem.phys_addr, dpa);
1634 gem.descriptor = descriptor;
1635 gem.type = type;
1636 gem.transaction_type = transaction_type;
1637
1638 if (has_channel) {
1639 gem.channel = channel;
1640 valid_flags |= CXL_GMER_VALID_CHANNEL;
1641 }
1642
1643 if (has_rank) {
1644 gem.rank = rank;
1645 valid_flags |= CXL_GMER_VALID_RANK;
1646 }
1647
1648 if (has_device) {
1649 st24_le_p(gem.device, device);
1650 valid_flags |= CXL_GMER_VALID_DEVICE;
1651 }
1652
1653 if (component_id) {
1654 strncpy((char *)gem.component_id, component_id,
1655 sizeof(gem.component_id) - 1);
1656 valid_flags |= CXL_GMER_VALID_COMPONENT;
1657 }
1658
1659 stw_le_p(&gem.validity_flags, valid_flags);
1660
1661 if (cxl_event_insert(cxlds, enc_log, (CXLEventRecordRaw *)&gem)) {
1662 cxl_event_irq_assert(ct3d);
1663 }
1664 }
1665
1666 #define CXL_DRAM_VALID_CHANNEL BIT(0)
1667 #define CXL_DRAM_VALID_RANK BIT(1)
1668 #define CXL_DRAM_VALID_NIBBLE_MASK BIT(2)
1669 #define CXL_DRAM_VALID_BANK_GROUP BIT(3)
1670 #define CXL_DRAM_VALID_BANK BIT(4)
1671 #define CXL_DRAM_VALID_ROW BIT(5)
1672 #define CXL_DRAM_VALID_COLUMN BIT(6)
1673 #define CXL_DRAM_VALID_CORRECTION_MASK BIT(7)
1674
qmp_cxl_inject_dram_event(const char * path,CxlEventLog log,uint8_t flags,uint64_t dpa,uint8_t descriptor,uint8_t type,uint8_t transaction_type,bool has_channel,uint8_t channel,bool has_rank,uint8_t rank,bool has_nibble_mask,uint32_t nibble_mask,bool has_bank_group,uint8_t bank_group,bool has_bank,uint8_t bank,bool has_row,uint32_t row,bool has_column,uint16_t column,bool has_correction_mask,uint64List * correction_mask,Error ** errp)1675 void qmp_cxl_inject_dram_event(const char *path, CxlEventLog log, uint8_t flags,
1676 uint64_t dpa, uint8_t descriptor,
1677 uint8_t type, uint8_t transaction_type,
1678 bool has_channel, uint8_t channel,
1679 bool has_rank, uint8_t rank,
1680 bool has_nibble_mask, uint32_t nibble_mask,
1681 bool has_bank_group, uint8_t bank_group,
1682 bool has_bank, uint8_t bank,
1683 bool has_row, uint32_t row,
1684 bool has_column, uint16_t column,
1685 bool has_correction_mask,
1686 uint64List *correction_mask,
1687 Error **errp)
1688 {
1689 Object *obj = object_resolve_path(path, NULL);
1690 CXLEventDram dram;
1691 CXLEventRecordHdr *hdr = &dram.hdr;
1692 CXLDeviceState *cxlds;
1693 CXLType3Dev *ct3d;
1694 uint16_t valid_flags = 0;
1695 uint8_t enc_log;
1696 int rc;
1697
1698 if (!obj) {
1699 error_setg(errp, "Unable to resolve path");
1700 return;
1701 }
1702 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1703 error_setg(errp, "Path does not point to a CXL type 3 device");
1704 return;
1705 }
1706 ct3d = CXL_TYPE3(obj);
1707 cxlds = &ct3d->cxl_dstate;
1708
1709 rc = ct3d_qmp_cxl_event_log_enc(log);
1710 if (rc < 0) {
1711 error_setg(errp, "Unhandled error log type");
1712 return;
1713 }
1714 enc_log = rc;
1715
1716 memset(&dram, 0, sizeof(dram));
1717 cxl_assign_event_header(hdr, &dram_uuid, flags, sizeof(dram),
1718 cxl_device_get_timestamp(&ct3d->cxl_dstate));
1719 stq_le_p(&dram.phys_addr, dpa);
1720 dram.descriptor = descriptor;
1721 dram.type = type;
1722 dram.transaction_type = transaction_type;
1723
1724 if (has_channel) {
1725 dram.channel = channel;
1726 valid_flags |= CXL_DRAM_VALID_CHANNEL;
1727 }
1728
1729 if (has_rank) {
1730 dram.rank = rank;
1731 valid_flags |= CXL_DRAM_VALID_RANK;
1732 }
1733
1734 if (has_nibble_mask) {
1735 st24_le_p(dram.nibble_mask, nibble_mask);
1736 valid_flags |= CXL_DRAM_VALID_NIBBLE_MASK;
1737 }
1738
1739 if (has_bank_group) {
1740 dram.bank_group = bank_group;
1741 valid_flags |= CXL_DRAM_VALID_BANK_GROUP;
1742 }
1743
1744 if (has_bank) {
1745 dram.bank = bank;
1746 valid_flags |= CXL_DRAM_VALID_BANK;
1747 }
1748
1749 if (has_row) {
1750 st24_le_p(dram.row, row);
1751 valid_flags |= CXL_DRAM_VALID_ROW;
1752 }
1753
1754 if (has_column) {
1755 stw_le_p(&dram.column, column);
1756 valid_flags |= CXL_DRAM_VALID_COLUMN;
1757 }
1758
1759 if (has_correction_mask) {
1760 int count = 0;
1761 while (correction_mask && count < 4) {
1762 stq_le_p(&dram.correction_mask[count],
1763 correction_mask->value);
1764 count++;
1765 correction_mask = correction_mask->next;
1766 }
1767 valid_flags |= CXL_DRAM_VALID_CORRECTION_MASK;
1768 }
1769
1770 stw_le_p(&dram.validity_flags, valid_flags);
1771
1772 if (cxl_event_insert(cxlds, enc_log, (CXLEventRecordRaw *)&dram)) {
1773 cxl_event_irq_assert(ct3d);
1774 }
1775 return;
1776 }
1777
qmp_cxl_inject_memory_module_event(const char * path,CxlEventLog log,uint8_t flags,uint8_t type,uint8_t health_status,uint8_t media_status,uint8_t additional_status,uint8_t life_used,int16_t temperature,uint32_t dirty_shutdown_count,uint32_t corrected_volatile_error_count,uint32_t corrected_persist_error_count,Error ** errp)1778 void qmp_cxl_inject_memory_module_event(const char *path, CxlEventLog log,
1779 uint8_t flags, uint8_t type,
1780 uint8_t health_status,
1781 uint8_t media_status,
1782 uint8_t additional_status,
1783 uint8_t life_used,
1784 int16_t temperature,
1785 uint32_t dirty_shutdown_count,
1786 uint32_t corrected_volatile_error_count,
1787 uint32_t corrected_persist_error_count,
1788 Error **errp)
1789 {
1790 Object *obj = object_resolve_path(path, NULL);
1791 CXLEventMemoryModule module;
1792 CXLEventRecordHdr *hdr = &module.hdr;
1793 CXLDeviceState *cxlds;
1794 CXLType3Dev *ct3d;
1795 uint8_t enc_log;
1796 int rc;
1797
1798 if (!obj) {
1799 error_setg(errp, "Unable to resolve path");
1800 return;
1801 }
1802 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) {
1803 error_setg(errp, "Path does not point to a CXL type 3 device");
1804 return;
1805 }
1806 ct3d = CXL_TYPE3(obj);
1807 cxlds = &ct3d->cxl_dstate;
1808
1809 rc = ct3d_qmp_cxl_event_log_enc(log);
1810 if (rc < 0) {
1811 error_setg(errp, "Unhandled error log type");
1812 return;
1813 }
1814 enc_log = rc;
1815
1816 memset(&module, 0, sizeof(module));
1817 cxl_assign_event_header(hdr, &memory_module_uuid, flags, sizeof(module),
1818 cxl_device_get_timestamp(&ct3d->cxl_dstate));
1819
1820 module.type = type;
1821 module.health_status = health_status;
1822 module.media_status = media_status;
1823 module.additional_status = additional_status;
1824 module.life_used = life_used;
1825 stw_le_p(&module.temperature, temperature);
1826 stl_le_p(&module.dirty_shutdown_count, dirty_shutdown_count);
1827 stl_le_p(&module.corrected_volatile_error_count,
1828 corrected_volatile_error_count);
1829 stl_le_p(&module.corrected_persistent_error_count,
1830 corrected_persist_error_count);
1831
1832 if (cxl_event_insert(cxlds, enc_log, (CXLEventRecordRaw *)&module)) {
1833 cxl_event_irq_assert(ct3d);
1834 }
1835 }
1836
1837 /* CXL r3.1 Table 8-50: Dynamic Capacity Event Record */
1838 static const QemuUUID dynamic_capacity_uuid = {
1839 .data = UUID(0xca95afa7, 0xf183, 0x4018, 0x8c, 0x2f,
1840 0x95, 0x26, 0x8e, 0x10, 0x1a, 0x2a),
1841 };
1842
1843 typedef enum CXLDCEventType {
1844 DC_EVENT_ADD_CAPACITY = 0x0,
1845 DC_EVENT_RELEASE_CAPACITY = 0x1,
1846 DC_EVENT_FORCED_RELEASE_CAPACITY = 0x2,
1847 DC_EVENT_REGION_CONFIG_UPDATED = 0x3,
1848 DC_EVENT_ADD_CAPACITY_RSP = 0x4,
1849 DC_EVENT_CAPACITY_RELEASED = 0x5,
1850 } CXLDCEventType;
1851
1852 /*
1853 * Check whether the range [dpa, dpa + len - 1] has overlaps with extents in
1854 * the list.
1855 * Return value: return true if has overlaps; otherwise, return false
1856 */
cxl_extents_overlaps_dpa_range(CXLDCExtentList * list,uint64_t dpa,uint64_t len)1857 static bool cxl_extents_overlaps_dpa_range(CXLDCExtentList *list,
1858 uint64_t dpa, uint64_t len)
1859 {
1860 CXLDCExtent *ent;
1861 Range range1, range2;
1862
1863 if (!list) {
1864 return false;
1865 }
1866
1867 range_init_nofail(&range1, dpa, len);
1868 QTAILQ_FOREACH(ent, list, node) {
1869 range_init_nofail(&range2, ent->start_dpa, ent->len);
1870 if (range_overlaps_range(&range1, &range2)) {
1871 return true;
1872 }
1873 }
1874 return false;
1875 }
1876
1877 /*
1878 * Check whether the range [dpa, dpa + len - 1] is contained by extents in
1879 * the list.
1880 * Will check multiple extents containment once superset release is added.
1881 * Return value: return true if range is contained; otherwise, return false
1882 */
cxl_extents_contains_dpa_range(CXLDCExtentList * list,uint64_t dpa,uint64_t len)1883 bool cxl_extents_contains_dpa_range(CXLDCExtentList *list,
1884 uint64_t dpa, uint64_t len)
1885 {
1886 CXLDCExtent *ent;
1887 Range range1, range2;
1888
1889 if (!list) {
1890 return false;
1891 }
1892
1893 range_init_nofail(&range1, dpa, len);
1894 QTAILQ_FOREACH(ent, list, node) {
1895 range_init_nofail(&range2, ent->start_dpa, ent->len);
1896 if (range_contains_range(&range2, &range1)) {
1897 return true;
1898 }
1899 }
1900 return false;
1901 }
1902
cxl_extent_groups_overlaps_dpa_range(CXLDCExtentGroupList * list,uint64_t dpa,uint64_t len)1903 static bool cxl_extent_groups_overlaps_dpa_range(CXLDCExtentGroupList *list,
1904 uint64_t dpa, uint64_t len)
1905 {
1906 CXLDCExtentGroup *group;
1907
1908 if (!list) {
1909 return false;
1910 }
1911
1912 QTAILQ_FOREACH(group, list, node) {
1913 if (cxl_extents_overlaps_dpa_range(&group->list, dpa, len)) {
1914 return true;
1915 }
1916 }
1917 return false;
1918 }
1919
1920 /*
1921 * The main function to process dynamic capacity event with extent list.
1922 * Currently DC extents add/release requests are processed.
1923 */
qmp_cxl_process_dynamic_capacity_prescriptive(const char * path,uint16_t hid,CXLDCEventType type,uint8_t rid,CxlDynamicCapacityExtentList * records,Error ** errp)1924 static void qmp_cxl_process_dynamic_capacity_prescriptive(const char *path,
1925 uint16_t hid, CXLDCEventType type, uint8_t rid,
1926 CxlDynamicCapacityExtentList *records, Error **errp)
1927 {
1928 Object *obj;
1929 CXLEventDynamicCapacity dCap = {};
1930 CXLEventRecordHdr *hdr = &dCap.hdr;
1931 CXLType3Dev *dcd;
1932 uint8_t flags = 1 << CXL_EVENT_TYPE_INFO;
1933 uint32_t num_extents = 0;
1934 CxlDynamicCapacityExtentList *list;
1935 CXLDCExtentGroup *group = NULL;
1936 g_autofree CXLDCExtentRaw *extents = NULL;
1937 uint8_t enc_log = CXL_EVENT_TYPE_DYNAMIC_CAP;
1938 uint64_t dpa, offset, len, block_size;
1939 g_autofree unsigned long *blk_bitmap = NULL;
1940 int i;
1941
1942 obj = object_resolve_path_type(path, TYPE_CXL_TYPE3, NULL);
1943 if (!obj) {
1944 error_setg(errp, "Unable to resolve CXL type 3 device");
1945 return;
1946 }
1947
1948 dcd = CXL_TYPE3(obj);
1949 if (!dcd->dc.num_regions) {
1950 error_setg(errp, "No dynamic capacity support from the device");
1951 return;
1952 }
1953
1954
1955 if (rid >= dcd->dc.num_regions) {
1956 error_setg(errp, "region id is too large");
1957 return;
1958 }
1959 block_size = dcd->dc.regions[rid].block_size;
1960 blk_bitmap = bitmap_new(dcd->dc.regions[rid].len / block_size);
1961
1962 /* Sanity check and count the extents */
1963 list = records;
1964 while (list) {
1965 offset = list->value->offset;
1966 len = list->value->len;
1967 dpa = offset + dcd->dc.regions[rid].base;
1968
1969 if (len == 0) {
1970 error_setg(errp, "extent with 0 length is not allowed");
1971 return;
1972 }
1973
1974 if (offset % block_size || len % block_size) {
1975 error_setg(errp, "dpa or len is not aligned to region block size");
1976 return;
1977 }
1978
1979 if (offset + len > dcd->dc.regions[rid].len) {
1980 error_setg(errp, "extent range is beyond the region end");
1981 return;
1982 }
1983
1984 /* No duplicate or overlapped extents are allowed */
1985 if (test_any_bits_set(blk_bitmap, offset / block_size,
1986 len / block_size)) {
1987 error_setg(errp, "duplicate or overlapped extents are detected");
1988 return;
1989 }
1990 bitmap_set(blk_bitmap, offset / block_size, len / block_size);
1991
1992 if (type == DC_EVENT_RELEASE_CAPACITY) {
1993 if (cxl_extent_groups_overlaps_dpa_range(&dcd->dc.extents_pending,
1994 dpa, len)) {
1995 error_setg(errp,
1996 "cannot release extent with pending DPA range");
1997 return;
1998 }
1999 if (!ct3_test_region_block_backed(dcd, dpa, len)) {
2000 error_setg(errp,
2001 "cannot release extent with non-existing DPA range");
2002 return;
2003 }
2004 } else if (type == DC_EVENT_ADD_CAPACITY) {
2005 if (cxl_extents_overlaps_dpa_range(&dcd->dc.extents, dpa, len)) {
2006 error_setg(errp,
2007 "cannot add DPA already accessible to the same LD");
2008 return;
2009 }
2010 if (cxl_extent_groups_overlaps_dpa_range(&dcd->dc.extents_pending,
2011 dpa, len)) {
2012 error_setg(errp,
2013 "cannot add DPA again while still pending");
2014 return;
2015 }
2016 }
2017 list = list->next;
2018 num_extents++;
2019 }
2020
2021 /* Create extent list for event being passed to host */
2022 i = 0;
2023 list = records;
2024 extents = g_new0(CXLDCExtentRaw, num_extents);
2025 while (list) {
2026 offset = list->value->offset;
2027 len = list->value->len;
2028 dpa = dcd->dc.regions[rid].base + offset;
2029
2030 extents[i].start_dpa = dpa;
2031 extents[i].len = len;
2032 memset(extents[i].tag, 0, 0x10);
2033 extents[i].shared_seq = 0;
2034 if (type == DC_EVENT_ADD_CAPACITY) {
2035 group = cxl_insert_extent_to_extent_group(group,
2036 extents[i].start_dpa,
2037 extents[i].len,
2038 extents[i].tag,
2039 extents[i].shared_seq);
2040 }
2041
2042 list = list->next;
2043 i++;
2044 }
2045 if (group) {
2046 cxl_extent_group_list_insert_tail(&dcd->dc.extents_pending, group);
2047 }
2048
2049 /*
2050 * CXL r3.1 section 8.2.9.2.1.6: Dynamic Capacity Event Record
2051 *
2052 * All Dynamic Capacity event records shall set the Event Record Severity
2053 * field in the Common Event Record Format to Informational Event. All
2054 * Dynamic Capacity related events shall be logged in the Dynamic Capacity
2055 * Event Log.
2056 */
2057 cxl_assign_event_header(hdr, &dynamic_capacity_uuid, flags, sizeof(dCap),
2058 cxl_device_get_timestamp(&dcd->cxl_dstate));
2059
2060 dCap.type = type;
2061 /* FIXME: for now, validity flag is cleared */
2062 dCap.validity_flags = 0;
2063 stw_le_p(&dCap.host_id, hid);
2064 /* only valid for DC_REGION_CONFIG_UPDATED event */
2065 dCap.updated_region_id = 0;
2066 for (i = 0; i < num_extents; i++) {
2067 memcpy(&dCap.dynamic_capacity_extent, &extents[i],
2068 sizeof(CXLDCExtentRaw));
2069
2070 dCap.flags = 0;
2071 if (i < num_extents - 1) {
2072 /* Set "More" flag */
2073 dCap.flags |= BIT(0);
2074 }
2075
2076 if (cxl_event_insert(&dcd->cxl_dstate, enc_log,
2077 (CXLEventRecordRaw *)&dCap)) {
2078 cxl_event_irq_assert(dcd);
2079 }
2080 }
2081 }
2082
qmp_cxl_add_dynamic_capacity(const char * path,uint16_t host_id,CxlExtentSelectionPolicy sel_policy,uint8_t region,const char * tag,CxlDynamicCapacityExtentList * extents,Error ** errp)2083 void qmp_cxl_add_dynamic_capacity(const char *path, uint16_t host_id,
2084 CxlExtentSelectionPolicy sel_policy,
2085 uint8_t region, const char *tag,
2086 CxlDynamicCapacityExtentList *extents,
2087 Error **errp)
2088 {
2089 switch (sel_policy) {
2090 case CXL_EXTENT_SELECTION_POLICY_PRESCRIPTIVE:
2091 qmp_cxl_process_dynamic_capacity_prescriptive(path, host_id,
2092 DC_EVENT_ADD_CAPACITY,
2093 region, extents, errp);
2094 return;
2095 default:
2096 error_setg(errp, "Selection policy not supported");
2097 return;
2098 }
2099 }
2100
qmp_cxl_release_dynamic_capacity(const char * path,uint16_t host_id,CxlExtentRemovalPolicy removal_policy,bool has_forced_removal,bool forced_removal,bool has_sanitize_on_release,bool sanitize_on_release,uint8_t region,const char * tag,CxlDynamicCapacityExtentList * extents,Error ** errp)2101 void qmp_cxl_release_dynamic_capacity(const char *path, uint16_t host_id,
2102 CxlExtentRemovalPolicy removal_policy,
2103 bool has_forced_removal,
2104 bool forced_removal,
2105 bool has_sanitize_on_release,
2106 bool sanitize_on_release,
2107 uint8_t region,
2108 const char *tag,
2109 CxlDynamicCapacityExtentList *extents,
2110 Error **errp)
2111 {
2112 CXLDCEventType type = DC_EVENT_RELEASE_CAPACITY;
2113
2114 if (has_forced_removal && forced_removal) {
2115 /* TODO: enable forced removal in the future */
2116 type = DC_EVENT_FORCED_RELEASE_CAPACITY;
2117 error_setg(errp, "Forced removal not supported yet");
2118 return;
2119 }
2120
2121 switch (removal_policy) {
2122 case CXL_EXTENT_REMOVAL_POLICY_PRESCRIPTIVE:
2123 qmp_cxl_process_dynamic_capacity_prescriptive(path, host_id, type,
2124 region, extents, errp);
2125 return;
2126 default:
2127 error_setg(errp, "Removal policy not supported");
2128 return;
2129 }
2130 }
2131
ct3_class_init(ObjectClass * oc,void * data)2132 static void ct3_class_init(ObjectClass *oc, void *data)
2133 {
2134 DeviceClass *dc = DEVICE_CLASS(oc);
2135 PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
2136 CXLType3Class *cvc = CXL_TYPE3_CLASS(oc);
2137
2138 pc->realize = ct3_realize;
2139 pc->exit = ct3_exit;
2140 pc->class_id = PCI_CLASS_MEMORY_CXL;
2141 pc->vendor_id = PCI_VENDOR_ID_INTEL;
2142 pc->device_id = 0xd93; /* LVF for now */
2143 pc->revision = 1;
2144
2145 pc->config_write = ct3d_config_write;
2146 pc->config_read = ct3d_config_read;
2147
2148 set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
2149 dc->desc = "CXL Memory Device (Type 3)";
2150 device_class_set_legacy_reset(dc, ct3d_reset);
2151 device_class_set_props(dc, ct3_props);
2152
2153 cvc->get_lsa_size = get_lsa_size;
2154 cvc->get_lsa = get_lsa;
2155 cvc->set_lsa = set_lsa;
2156 cvc->set_cacheline = set_cacheline;
2157 }
2158
2159 static const TypeInfo ct3d_info = {
2160 .name = TYPE_CXL_TYPE3,
2161 .parent = TYPE_PCI_DEVICE,
2162 .class_size = sizeof(struct CXLType3Class),
2163 .class_init = ct3_class_init,
2164 .instance_size = sizeof(CXLType3Dev),
2165 .interfaces = (InterfaceInfo[]) {
2166 { INTERFACE_CXL_DEVICE },
2167 { INTERFACE_PCIE_DEVICE },
2168 {}
2169 },
2170 };
2171
ct3d_registers(void)2172 static void ct3d_registers(void)
2173 {
2174 type_register_static(&ct3d_info);
2175 }
2176
2177 type_init(ct3d_registers);
2178