1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ 3 #include <linux/libnvdimm.h> 4 #include <asm/unaligned.h> 5 #include <linux/device.h> 6 #include <linux/module.h> 7 #include <linux/ndctl.h> 8 #include <linux/async.h> 9 #include <linux/slab.h> 10 #include <linux/nd.h> 11 #include "cxlmem.h" 12 #include "cxl.h" 13 14 extern const struct nvdimm_security_ops *cxl_security_ops; 15 16 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX); 17 18 static void clear_exclusive(void *cxlds) 19 { 20 clear_exclusive_cxl_commands(cxlds, exclusive_cmds); 21 } 22 23 static void unregister_nvdimm(void *nvdimm) 24 { 25 nvdimm_delete(nvdimm); 26 } 27 28 static ssize_t provider_show(struct device *dev, struct device_attribute *attr, char *buf) 29 { 30 struct nvdimm *nvdimm = to_nvdimm(dev); 31 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); 32 33 return sysfs_emit(buf, "%s\n", dev_name(&cxl_nvd->dev)); 34 } 35 static DEVICE_ATTR_RO(provider); 36 37 static ssize_t id_show(struct device *dev, struct device_attribute *attr, char *buf) 38 { 39 struct nvdimm *nvdimm = to_nvdimm(dev); 40 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); 41 struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds; 42 43 return sysfs_emit(buf, "%lld\n", cxlds->serial); 44 } 45 static DEVICE_ATTR_RO(id); 46 47 static struct attribute *cxl_dimm_attributes[] = { 48 &dev_attr_id.attr, 49 &dev_attr_provider.attr, 50 NULL 51 }; 52 53 static const struct attribute_group cxl_dimm_attribute_group = { 54 .name = "cxl", 55 .attrs = cxl_dimm_attributes, 56 }; 57 58 static const struct attribute_group *cxl_dimm_attribute_groups[] = { 59 &cxl_dimm_attribute_group, 60 NULL 61 }; 62 63 static int cxl_nvdimm_probe(struct device *dev) 64 { 65 struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev); 66 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; 67 struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb; 68 unsigned long flags = 0, cmd_mask = 0; 69 struct cxl_dev_state *cxlds = cxlmd->cxlds; 70 struct nvdimm *nvdimm; 71 int rc; 72 73 set_exclusive_cxl_commands(cxlds, exclusive_cmds); 74 rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds); 75 if (rc) 76 return rc; 77 78 set_bit(NDD_LABELING, &flags); 79 set_bit(NDD_REGISTER_SYNC, &flags); 80 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask); 81 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask); 82 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask); 83 nvdimm = __nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd, 84 cxl_dimm_attribute_groups, flags, 85 cmd_mask, 0, NULL, cxl_nvd->dev_id, 86 cxl_security_ops, NULL); 87 if (!nvdimm) 88 return -ENOMEM; 89 90 dev_set_drvdata(dev, nvdimm); 91 return devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm); 92 } 93 94 static struct cxl_driver cxl_nvdimm_driver = { 95 .name = "cxl_nvdimm", 96 .probe = cxl_nvdimm_probe, 97 .id = CXL_DEVICE_NVDIMM, 98 .drv = { 99 .suppress_bind_attrs = true, 100 }, 101 }; 102 103 static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds, 104 struct nd_cmd_get_config_size *cmd, 105 unsigned int buf_len) 106 { 107 if (sizeof(*cmd) > buf_len) 108 return -EINVAL; 109 110 *cmd = (struct nd_cmd_get_config_size) { 111 .config_size = cxlds->lsa_size, 112 .max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa), 113 }; 114 115 return 0; 116 } 117 118 static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds, 119 struct nd_cmd_get_config_data_hdr *cmd, 120 unsigned int buf_len) 121 { 122 struct cxl_mbox_get_lsa get_lsa; 123 struct cxl_mbox_cmd mbox_cmd; 124 int rc; 125 126 if (sizeof(*cmd) > buf_len) 127 return -EINVAL; 128 if (struct_size(cmd, out_buf, cmd->in_length) > buf_len) 129 return -EINVAL; 130 131 get_lsa = (struct cxl_mbox_get_lsa) { 132 .offset = cpu_to_le32(cmd->in_offset), 133 .length = cpu_to_le32(cmd->in_length), 134 }; 135 mbox_cmd = (struct cxl_mbox_cmd) { 136 .opcode = CXL_MBOX_OP_GET_LSA, 137 .payload_in = &get_lsa, 138 .size_in = sizeof(get_lsa), 139 .size_out = cmd->in_length, 140 .payload_out = cmd->out_buf, 141 }; 142 143 rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); 144 cmd->status = 0; 145 146 return rc; 147 } 148 149 static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds, 150 struct nd_cmd_set_config_hdr *cmd, 151 unsigned int buf_len) 152 { 153 struct cxl_mbox_set_lsa *set_lsa; 154 struct cxl_mbox_cmd mbox_cmd; 155 int rc; 156 157 if (sizeof(*cmd) > buf_len) 158 return -EINVAL; 159 160 /* 4-byte status follows the input data in the payload */ 161 if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len) 162 return -EINVAL; 163 164 set_lsa = 165 kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL); 166 if (!set_lsa) 167 return -ENOMEM; 168 169 *set_lsa = (struct cxl_mbox_set_lsa) { 170 .offset = cpu_to_le32(cmd->in_offset), 171 }; 172 memcpy(set_lsa->data, cmd->in_buf, cmd->in_length); 173 mbox_cmd = (struct cxl_mbox_cmd) { 174 .opcode = CXL_MBOX_OP_SET_LSA, 175 .payload_in = set_lsa, 176 .size_in = struct_size(set_lsa, data, cmd->in_length), 177 }; 178 179 rc = cxl_internal_send_cmd(cxlds, &mbox_cmd); 180 181 /* 182 * Set "firmware" status (4-packed bytes at the end of the input 183 * payload. 184 */ 185 put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]); 186 kvfree(set_lsa); 187 188 return rc; 189 } 190 191 static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd, 192 void *buf, unsigned int buf_len) 193 { 194 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); 195 unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm); 196 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd; 197 struct cxl_dev_state *cxlds = cxlmd->cxlds; 198 199 if (!test_bit(cmd, &cmd_mask)) 200 return -ENOTTY; 201 202 switch (cmd) { 203 case ND_CMD_GET_CONFIG_SIZE: 204 return cxl_pmem_get_config_size(cxlds, buf, buf_len); 205 case ND_CMD_GET_CONFIG_DATA: 206 return cxl_pmem_get_config_data(cxlds, buf, buf_len); 207 case ND_CMD_SET_CONFIG_DATA: 208 return cxl_pmem_set_config_data(cxlds, buf, buf_len); 209 default: 210 return -ENOTTY; 211 } 212 } 213 214 static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc, 215 struct nvdimm *nvdimm, unsigned int cmd, void *buf, 216 unsigned int buf_len, int *cmd_rc) 217 { 218 /* 219 * No firmware response to translate, let the transport error 220 * code take precedence. 221 */ 222 *cmd_rc = 0; 223 224 if (!nvdimm) 225 return -ENOTTY; 226 return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len); 227 } 228 229 static int detach_nvdimm(struct device *dev, void *data) 230 { 231 struct cxl_nvdimm *cxl_nvd; 232 bool release = false; 233 234 if (!is_cxl_nvdimm(dev)) 235 return 0; 236 237 device_lock(dev); 238 if (!dev->driver) 239 goto out; 240 241 cxl_nvd = to_cxl_nvdimm(dev); 242 if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data) 243 release = true; 244 out: 245 device_unlock(dev); 246 if (release) 247 device_release_driver(dev); 248 return 0; 249 } 250 251 static void unregister_nvdimm_bus(void *_cxl_nvb) 252 { 253 struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb; 254 struct nvdimm_bus *nvdimm_bus = cxl_nvb->nvdimm_bus; 255 256 bus_for_each_dev(&cxl_bus_type, NULL, cxl_nvb, detach_nvdimm); 257 258 cxl_nvb->nvdimm_bus = NULL; 259 nvdimm_bus_unregister(nvdimm_bus); 260 } 261 262 static int cxl_nvdimm_bridge_probe(struct device *dev) 263 { 264 struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev); 265 266 cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) { 267 .provider_name = "CXL", 268 .module = THIS_MODULE, 269 .ndctl = cxl_pmem_ctl, 270 }; 271 272 cxl_nvb->nvdimm_bus = 273 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc); 274 275 if (!cxl_nvb->nvdimm_bus) 276 return -ENOMEM; 277 278 return devm_add_action_or_reset(dev, unregister_nvdimm_bus, cxl_nvb); 279 } 280 281 static struct cxl_driver cxl_nvdimm_bridge_driver = { 282 .name = "cxl_nvdimm_bridge", 283 .probe = cxl_nvdimm_bridge_probe, 284 .id = CXL_DEVICE_NVDIMM_BRIDGE, 285 .drv = { 286 .suppress_bind_attrs = true, 287 }, 288 }; 289 290 static void unregister_nvdimm_region(void *nd_region) 291 { 292 nvdimm_region_delete(nd_region); 293 } 294 295 static void cxlr_pmem_remove_resource(void *res) 296 { 297 remove_resource(res); 298 } 299 300 struct cxl_pmem_region_info { 301 u64 offset; 302 u64 serial; 303 }; 304 305 static int cxl_pmem_region_probe(struct device *dev) 306 { 307 struct nd_mapping_desc mappings[CXL_DECODER_MAX_INTERLEAVE]; 308 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); 309 struct cxl_region *cxlr = cxlr_pmem->cxlr; 310 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 311 struct cxl_pmem_region_info *info = NULL; 312 struct nd_interleave_set *nd_set; 313 struct nd_region_desc ndr_desc; 314 struct cxl_nvdimm *cxl_nvd; 315 struct nvdimm *nvdimm; 316 struct resource *res; 317 int rc, i = 0; 318 319 memset(&mappings, 0, sizeof(mappings)); 320 memset(&ndr_desc, 0, sizeof(ndr_desc)); 321 322 res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL); 323 if (!res) 324 return -ENOMEM; 325 326 res->name = "Persistent Memory"; 327 res->start = cxlr_pmem->hpa_range.start; 328 res->end = cxlr_pmem->hpa_range.end; 329 res->flags = IORESOURCE_MEM; 330 res->desc = IORES_DESC_PERSISTENT_MEMORY; 331 332 rc = insert_resource(&iomem_resource, res); 333 if (rc) 334 return rc; 335 336 rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res); 337 if (rc) 338 return rc; 339 340 ndr_desc.res = res; 341 ndr_desc.provider_data = cxlr_pmem; 342 343 ndr_desc.numa_node = memory_add_physaddr_to_nid(res->start); 344 ndr_desc.target_node = phys_to_target_node(res->start); 345 if (ndr_desc.target_node == NUMA_NO_NODE) { 346 ndr_desc.target_node = ndr_desc.numa_node; 347 dev_dbg(&cxlr->dev, "changing target node from %d to %d", 348 NUMA_NO_NODE, ndr_desc.target_node); 349 } 350 351 nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); 352 if (!nd_set) 353 return -ENOMEM; 354 355 ndr_desc.memregion = cxlr->id; 356 set_bit(ND_REGION_CXL, &ndr_desc.flags); 357 set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc.flags); 358 359 info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL); 360 if (!info) 361 return -ENOMEM; 362 363 for (i = 0; i < cxlr_pmem->nr_mappings; i++) { 364 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; 365 struct cxl_memdev *cxlmd = m->cxlmd; 366 struct cxl_dev_state *cxlds = cxlmd->cxlds; 367 368 cxl_nvd = cxlmd->cxl_nvd; 369 nvdimm = dev_get_drvdata(&cxl_nvd->dev); 370 if (!nvdimm) { 371 dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i, 372 dev_name(&cxlmd->dev)); 373 rc = -ENODEV; 374 goto out_nvd; 375 } 376 377 m->cxl_nvd = cxl_nvd; 378 mappings[i] = (struct nd_mapping_desc) { 379 .nvdimm = nvdimm, 380 .start = m->start, 381 .size = m->size, 382 .position = i, 383 }; 384 info[i].offset = m->start; 385 info[i].serial = cxlds->serial; 386 } 387 ndr_desc.num_mappings = cxlr_pmem->nr_mappings; 388 ndr_desc.mapping = mappings; 389 390 /* 391 * TODO enable CXL labels which skip the need for 'interleave-set cookie' 392 */ 393 nd_set->cookie1 = 394 nd_fletcher64(info, sizeof(*info) * cxlr_pmem->nr_mappings, 0); 395 nd_set->cookie2 = nd_set->cookie1; 396 ndr_desc.nd_set = nd_set; 397 398 cxlr_pmem->nd_region = 399 nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc); 400 if (!cxlr_pmem->nd_region) { 401 rc = -ENOMEM; 402 goto out_nvd; 403 } 404 405 rc = devm_add_action_or_reset(dev, unregister_nvdimm_region, 406 cxlr_pmem->nd_region); 407 out_nvd: 408 kfree(info); 409 410 return rc; 411 } 412 413 static struct cxl_driver cxl_pmem_region_driver = { 414 .name = "cxl_pmem_region", 415 .probe = cxl_pmem_region_probe, 416 .id = CXL_DEVICE_PMEM_REGION, 417 .drv = { 418 .suppress_bind_attrs = true, 419 }, 420 }; 421 422 static __init int cxl_pmem_init(void) 423 { 424 int rc; 425 426 set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds); 427 set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds); 428 429 rc = cxl_driver_register(&cxl_nvdimm_bridge_driver); 430 if (rc) 431 return rc; 432 433 rc = cxl_driver_register(&cxl_nvdimm_driver); 434 if (rc) 435 goto err_nvdimm; 436 437 rc = cxl_driver_register(&cxl_pmem_region_driver); 438 if (rc) 439 goto err_region; 440 441 return 0; 442 443 err_region: 444 cxl_driver_unregister(&cxl_nvdimm_driver); 445 err_nvdimm: 446 cxl_driver_unregister(&cxl_nvdimm_bridge_driver); 447 return rc; 448 } 449 450 static __exit void cxl_pmem_exit(void) 451 { 452 cxl_driver_unregister(&cxl_pmem_region_driver); 453 cxl_driver_unregister(&cxl_nvdimm_driver); 454 cxl_driver_unregister(&cxl_nvdimm_bridge_driver); 455 } 456 457 MODULE_LICENSE("GPL v2"); 458 module_init(cxl_pmem_init); 459 module_exit(cxl_pmem_exit); 460 MODULE_IMPORT_NS(CXL); 461 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE); 462 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM); 463 MODULE_ALIAS_CXL(CXL_DEVICE_PMEM_REGION); 464