132 	struct papr_scm_priv *p = nd_region_provider_data(nd_region);  in papr_scm_pmem_flush()  local
136 	dev_dbg(&p->pdev->dev, "flush drc 0x%x", p->drc_index);  in papr_scm_pmem_flush()
139 		rc = plpar_hcall(H_SCM_FLUSH, ret_buf, p->drc_index, token);  in papr_scm_pmem_flush()
152 		dev_err(&p->pdev->dev, "flush error: %ld", rc);  in papr_scm_pmem_flush()
155 		dev_dbg(&p->pdev->dev, "flush drc 0x%x complete", p->drc_index);  in papr_scm_pmem_flush()
164 static int drc_pmem_bind(struct papr_scm_priv *p)  in drc_pmem_bind()  argument
180 		rc = plpar_hcall(H_SCM_BIND_MEM, ret, p->drc_index, 0,  in drc_pmem_bind()
181 				p->blocks, BIND_ANY_ADDR, token);  in drc_pmem_bind()
191 	p->bound_addr = saved;  in drc_pmem_bind()
192 	dev_dbg(&p->pdev->dev, "bound drc 0x%x to 0x%lx\n",  in drc_pmem_bind()
193 		p->drc_index, (unsigned long)saved);  in drc_pmem_bind()
197 static void drc_pmem_unbind(struct papr_scm_priv *p)  in drc_pmem_unbind()  argument
203 	dev_dbg(&p->pdev->dev, "unbind drc 0x%x\n", p->drc_index);  in drc_pmem_unbind()
210 				 p->drc_index, token);  in drc_pmem_unbind()
224 		dev_err(&p->pdev->dev, "unbind error: %lld\n", rc);  in drc_pmem_unbind()
226 		dev_dbg(&p->pdev->dev, "unbind drc 0x%x complete\n",  in drc_pmem_unbind()
227 			p->drc_index);  in drc_pmem_unbind()
232 static int drc_pmem_query_n_bind(struct papr_scm_priv *p)  in drc_pmem_query_n_bind()  argument
241 			 p->drc_index, 0);  in drc_pmem_query_n_bind()
248 			 p->drc_index, p->blocks - 1);  in drc_pmem_query_n_bind()
253 	if ((end_addr - start_addr) != ((p->blocks - 1) * p->block_size))  in drc_pmem_query_n_bind()
256 	p->bound_addr = start_addr;  in drc_pmem_query_n_bind()
257 	dev_dbg(&p->pdev->dev, "bound drc 0x%x to 0x%lx\n", p->drc_index, start_addr);  in drc_pmem_query_n_bind()
261 	dev_info(&p->pdev->dev,  in drc_pmem_query_n_bind()
263 	drc_pmem_unbind(p);  in drc_pmem_query_n_bind()
264 	return drc_pmem_bind(p);  in drc_pmem_query_n_bind()
279 static ssize_t drc_pmem_query_stats(struct papr_scm_priv *p,  in drc_pmem_query_stats()  argument
305 			size = p->stat_buffer_len;  in drc_pmem_query_stats()
312 	rc = plpar_hcall(H_SCM_PERFORMANCE_STATS, ret, p->drc_index,  in drc_pmem_query_stats()
318 		dev_err(&p->pdev->dev,  in drc_pmem_query_stats()
322 		dev_info(&p->pdev->dev,  in drc_pmem_query_stats()
326 		dev_dbg(&p->pdev->dev, "Performance stats unsupported\n");  in drc_pmem_query_stats()
329 		dev_err(&p->pdev->dev,  in drc_pmem_query_stats()
335 		dev_dbg(&p->pdev->dev,  in drc_pmem_query_stats()
341 	dev_dbg(&p->pdev->dev,  in drc_pmem_query_stats()
373 	struct papr_scm_priv *p = dev_get_drvdata(dev);  in papr_scm_pmu_get_value()  local
384 	if (!p)  in papr_scm_pmu_get_value()
397 	rc = drc_pmem_query_stats(p, stats, 1);  in papr_scm_pmu_get_value()
411 	struct papr_scm_priv *p;  in papr_scm_pmu_event_init()  local
428 	p = (struct papr_scm_priv *)nd_pmu->dev->driver_data;  in papr_scm_pmu_event_init()
429 	if (!p)  in papr_scm_pmu_event_init()
481 static void papr_scm_pmu_register(struct papr_scm_priv *p)  in papr_scm_pmu_register()  argument
492 	if (!p->stat_buffer_len) {  in papr_scm_pmu_register()
498 	nd_pmu->pmu.name = nvdimm_name(p->nvdimm);  in papr_scm_pmu_register()
508 	nodeid = numa_map_to_online_node(dev_to_node(&p->pdev->dev));  in papr_scm_pmu_register()
511 	rc = register_nvdimm_pmu(nd_pmu, p->pdev);  in papr_scm_pmu_register()
519 	p->pdev->archdata.priv = nd_pmu;  in papr_scm_pmu_register()
525 	dev_info(&p->pdev->dev, "nvdimm pmu didn't register rc=%d\n", rc);  in papr_scm_pmu_register()
529 static void papr_scm_pmu_register(struct papr_scm_priv *p) { }  in papr_scm_pmu_register()  argument
536 static int __drc_pmem_query_health(struct papr_scm_priv *p)  in __drc_pmem_query_health()  argument
543 	rc = plpar_hcall(H_SCM_HEALTH, ret, p->drc_index);  in __drc_pmem_query_health()
547 		dev_info_once(&p->pdev->dev,  in __drc_pmem_query_health()
551 		dev_err(&p->pdev->dev,  in __drc_pmem_query_health()
556 	p->lasthealth_jiffies = jiffies;  in __drc_pmem_query_health()
558 	if (p->health_bitmap_inject_mask)  in __drc_pmem_query_health()
559 		bitmap = (bitmap & ~p->health_bitmap_inject_mask) |  in __drc_pmem_query_health()
560 			p->health_bitmap_inject_mask;  in __drc_pmem_query_health()
561 	WRITE_ONCE(p->health_bitmap, bitmap);  in __drc_pmem_query_health()
562 	dev_dbg(&p->pdev->dev,  in __drc_pmem_query_health()
573 static int drc_pmem_query_health(struct papr_scm_priv *p)  in drc_pmem_query_health()  argument
579 	rc = mutex_lock_interruptible(&p->health_mutex);  in drc_pmem_query_health()
584 	cache_timeout = p->lasthealth_jiffies +  in drc_pmem_query_health()
589 		rc = __drc_pmem_query_health(p);  in drc_pmem_query_health()
594 	mutex_unlock(&p->health_mutex);  in drc_pmem_query_health()
598 static int papr_scm_meta_get(struct papr_scm_priv *p,  in papr_scm_meta_get()  argument
606 	if ((hdr->in_offset + hdr->in_length) > p->metadata_size)  in papr_scm_meta_get()
623 		ret = plpar_hcall(H_SCM_READ_METADATA, data, p->drc_index,  in papr_scm_meta_get()
651 static int papr_scm_meta_set(struct papr_scm_priv *p,  in papr_scm_meta_set()  argument
660 	if ((hdr->in_offset + hdr->in_length) > p->metadata_size)  in papr_scm_meta_set()
688 		ret = plpar_hcall_norets(H_SCM_WRITE_METADATA, p->drc_index,  in papr_scm_meta_set()
709 	struct papr_scm_priv *p;  in is_cmd_valid()  local
717 	p = nvdimm_provider_data(nvdimm);  in is_cmd_valid()
720 		dev_dbg(&p->pdev->dev, "Unsupported cmd=%u\n", cmd);  in is_cmd_valid()
729 			dev_dbg(&p->pdev->dev, "Invalid pkg size=%u\n",  in is_cmd_valid()
738 			dev_dbg(&p->pdev->dev, "Invalid pkg family=0x%llx\n",  in is_cmd_valid()
747 			dev_dbg(&p->pdev->dev, "PDSM[0x%x]: Invalid PDSM\n",  in is_cmd_valid()
754 			dev_dbg(&p->pdev->dev, "PDSM[0x%x]: Invalid payload\n",  in is_cmd_valid()
764 static int papr_pdsm_fuel_gauge(struct papr_scm_priv *p,  in papr_pdsm_fuel_gauge()  argument
773 	if (!p->stat_buffer_len)  in papr_pdsm_fuel_gauge()
789 	rc = drc_pmem_query_stats(p, stats, 1);  in papr_pdsm_fuel_gauge()
791 		dev_dbg(&p->pdev->dev, "Err(%d) fetching fuel gauge\n", rc);  in papr_pdsm_fuel_gauge()
796 	dev_dbg(&p->pdev->dev,  in papr_pdsm_fuel_gauge()
810 static int papr_pdsm_dsc(struct papr_scm_priv *p,  in papr_pdsm_dsc()  argument
814 	payload->health.dimm_dsc = p->dirty_shutdown_counter;  in papr_pdsm_dsc()
820 static int papr_pdsm_health(struct papr_scm_priv *p,  in papr_pdsm_health()  argument
826 	rc = mutex_lock_interruptible(&p->health_mutex);  in papr_pdsm_health()
831 	rc = __drc_pmem_query_health(p);  in papr_pdsm_health()
833 		mutex_unlock(&p->health_mutex);  in papr_pdsm_health()
840 		.dimm_unarmed = !!(p->health_bitmap & PAPR_PMEM_UNARMED_MASK),  in papr_pdsm_health()
841 		.dimm_bad_shutdown = !!(p->health_bitmap & PAPR_PMEM_BAD_SHUTDOWN_MASK),  in papr_pdsm_health()
842 		.dimm_bad_restore = !!(p->health_bitmap & PAPR_PMEM_BAD_RESTORE_MASK),  in papr_pdsm_health()
843 		.dimm_scrubbed = !!(p->health_bitmap & PAPR_PMEM_SCRUBBED_AND_LOCKED),  in papr_pdsm_health()
844 		.dimm_locked = !!(p->health_bitmap & PAPR_PMEM_SCRUBBED_AND_LOCKED),  in papr_pdsm_health()
845 		.dimm_encrypted = !!(p->health_bitmap & PAPR_PMEM_ENCRYPTED),  in papr_pdsm_health()
850 	if (p->health_bitmap & PAPR_PMEM_HEALTH_FATAL)  in papr_pdsm_health()
852 	else if (p->health_bitmap & PAPR_PMEM_HEALTH_CRITICAL)  in papr_pdsm_health()
854 	else if (p->health_bitmap & PAPR_PMEM_HEALTH_UNHEALTHY)  in papr_pdsm_health()
858 	mutex_unlock(&p->health_mutex);  in papr_pdsm_health()
861 	papr_pdsm_fuel_gauge(p, payload);  in papr_pdsm_health()
863 	papr_pdsm_dsc(p, payload);  in papr_pdsm_health()
872 static int papr_pdsm_smart_inject(struct papr_scm_priv *p,  in papr_pdsm_smart_inject()  argument
897 	dev_dbg(&p->pdev->dev, "[Smart-inject] inject_mask=%#llx clear_mask=%#llx\n",  in papr_pdsm_smart_inject()
901 	rc = mutex_lock_interruptible(&p->health_mutex);  in papr_pdsm_smart_inject()
906 	mask = READ_ONCE(p->health_bitmap_inject_mask);  in papr_pdsm_smart_inject()
908 	WRITE_ONCE(p->health_bitmap_inject_mask, mask);  in papr_pdsm_smart_inject()
911 	p->lasthealth_jiffies = 0;  in papr_pdsm_smart_inject()
913 	mutex_unlock(&p->health_mutex);  in papr_pdsm_smart_inject()
981 static int papr_scm_service_pdsm(struct papr_scm_priv *p,  in papr_scm_service_pdsm()  argument
996 		dev_dbg(&p->pdev->dev, "PDSM[0x%x]: Invalid reserved field\n",  in papr_scm_service_pdsm()
1004 		dev_dbg(&p->pdev->dev, "PDSM[0x%x]: Mismatched size_in=%d\n",  in papr_scm_service_pdsm()
1012 		dev_dbg(&p->pdev->dev, "PDSM[0x%x]: Mismatched size_out=%d\n",  in papr_scm_service_pdsm()
1019 		dev_dbg(&p->pdev->dev, "PDSM[0x%x]: Servicing..\n", pdsm);  in papr_scm_service_pdsm()
1021 		rc = pdsc->service(p, &pdsm_pkg->payload);  in papr_scm_service_pdsm()
1033 		dev_dbg(&p->pdev->dev, "PDSM[0x%x]: Unsupported PDSM request\n",  in papr_scm_service_pdsm()
1048 	struct papr_scm_priv *p;  in papr_scm_ndctl()  local
1061 	p = nvdimm_provider_data(nvdimm);  in papr_scm_ndctl()
1069 		get_size_hdr->config_size = p->metadata_size;  in papr_scm_ndctl()
1074 		*cmd_rc = papr_scm_meta_get(p, buf);  in papr_scm_ndctl()
1078 		*cmd_rc = papr_scm_meta_set(p, buf);  in papr_scm_ndctl()
1083 		*cmd_rc = papr_scm_service_pdsm(p, call_pkg);  in papr_scm_ndctl()
1087 		dev_dbg(&p->pdev->dev, "Unknown command = %d\n", cmd);  in papr_scm_ndctl()
1091 	dev_dbg(&p->pdev->dev, "returned with cmd_rc = %d\n", *cmd_rc);  in papr_scm_ndctl()
1101 	struct papr_scm_priv *p = nvdimm_provider_data(dimm);  in health_bitmap_inject_show()  local
1104 		       READ_ONCE(p->health_bitmap_inject_mask));  in health_bitmap_inject_show()
1118 	struct papr_scm_priv *p = nvdimm_provider_data(dimm);  in perf_stats_show()  local
1120 	if (!p->stat_buffer_len)  in perf_stats_show()
1124 	stats = kzalloc(p->stat_buffer_len, GFP_KERNEL);  in perf_stats_show()
1129 	rc = drc_pmem_query_stats(p, stats, 0);  in perf_stats_show()
1156 	struct papr_scm_priv *p = nvdimm_provider_data(dimm);  in flags_show()  local
1161 	rc = drc_pmem_query_health(p);  in flags_show()
1166 	health = READ_ONCE(p->health_bitmap);  in flags_show()
1198 	struct papr_scm_priv *p = nvdimm_provider_data(dimm);  in dirty_shutdown_show()  local
1200 	return sysfs_emit(buf, "%llu\n", p->dirty_shutdown_counter);  in dirty_shutdown_show()
1209 	struct papr_scm_priv *p = nvdimm_provider_data(nvdimm);  in papr_nd_attribute_visible()  local
1212 	if (attr == &dev_attr_perf_stats.attr && p->stat_buffer_len == 0)  in papr_nd_attribute_visible()
1238 static int papr_scm_nvdimm_init(struct papr_scm_priv *p)  in papr_scm_nvdimm_init()  argument
1240 	struct device *dev = &p->pdev->dev;  in papr_scm_nvdimm_init()
1246 	p->bus_desc.ndctl = papr_scm_ndctl;  in papr_scm_nvdimm_init()
1247 	p->bus_desc.module = THIS_MODULE;  in papr_scm_nvdimm_init()
1248 	p->bus_desc.of_node = p->pdev->dev.of_node;  in papr_scm_nvdimm_init()
1249 	p->bus_desc.provider_name = kstrdup(p->pdev->name, GFP_KERNEL);  in papr_scm_nvdimm_init()
1252 	set_bit(NVDIMM_FAMILY_PAPR, &p->bus_desc.dimm_family_mask);  in papr_scm_nvdimm_init()
1254 	if (!p->bus_desc.provider_name)  in papr_scm_nvdimm_init()
1257 	p->bus = nvdimm_bus_register(NULL, &p->bus_desc);  in papr_scm_nvdimm_init()
1258 	if (!p->bus) {  in papr_scm_nvdimm_init()
1259 		dev_err(dev, "Error creating nvdimm bus %pOF\n", p->dn);  in papr_scm_nvdimm_init()
1260 		kfree(p->bus_desc.provider_name);  in papr_scm_nvdimm_init()
1271 	__drc_pmem_query_health(p);  in papr_scm_nvdimm_init()
1273 	if (p->health_bitmap & PAPR_PMEM_UNARMED_MASK)  in papr_scm_nvdimm_init()
1276 	p->nvdimm = nvdimm_create(p->bus, p, papr_nd_attr_groups,  in papr_scm_nvdimm_init()
1278 	if (!p->nvdimm) {  in papr_scm_nvdimm_init()
1279 		dev_err(dev, "Error creating DIMM object for %pOF\n", p->dn);  in papr_scm_nvdimm_init()
1283 	if (nvdimm_bus_check_dimm_count(p->bus, 1))  in papr_scm_nvdimm_init()
1289 	mapping.nvdimm = p->nvdimm;  in papr_scm_nvdimm_init()
1291 	mapping.size = p->blocks * p->block_size; // XXX: potential overflow?  in papr_scm_nvdimm_init()
1294 	target_nid = dev_to_node(&p->pdev->dev);  in papr_scm_nvdimm_init()
1298 	ndr_desc.res = &p->res;  in papr_scm_nvdimm_init()
1299 	ndr_desc.of_node = p->dn;  in papr_scm_nvdimm_init()
1300 	ndr_desc.provider_data = p;  in papr_scm_nvdimm_init()
1303 	ndr_desc.nd_set = &p->nd_set;  in papr_scm_nvdimm_init()
1305 	if (p->hcall_flush_required) {  in papr_scm_nvdimm_init()
1310 	if (p->is_volatile)  in papr_scm_nvdimm_init()
1311 		p->region = nvdimm_volatile_region_create(p->bus, &ndr_desc);  in papr_scm_nvdimm_init()
1314 		p->region = nvdimm_pmem_region_create(p->bus, &ndr_desc);  in papr_scm_nvdimm_init()
1316 	if (!p->region) {  in papr_scm_nvdimm_init()
1318 				ndr_desc.res, p->dn);  in papr_scm_nvdimm_init()
1326 	list_add_tail(&p->region_list, &papr_nd_regions);  in papr_scm_nvdimm_init()
1331 err:	nvdimm_bus_unregister(p->bus);  in papr_scm_nvdimm_init()
1332 	kfree(p->bus_desc.provider_name);  in papr_scm_nvdimm_init()
1356 	struct papr_scm_priv *p;  in handle_mce_ue()  local
1379 	list_for_each_entry(p, &papr_nd_regions, region_list) {  in handle_mce_ue()
1380 		if (phys_addr >= p->res.start && phys_addr <= p->res.end) {  in handle_mce_ue()
1387 		papr_scm_add_badblock(p->region, p->bus, phys_addr);  in handle_mce_ue()
1403 	struct papr_scm_priv *p;  in papr_scm_probe()  local
1439 	p = kzalloc(sizeof(*p), GFP_KERNEL);  in papr_scm_probe()
1440 	if (!p)  in papr_scm_probe()
1444 	mutex_init(&p->health_mutex);  in papr_scm_probe()
1449 	p->dn = dn;  in papr_scm_probe()
1450 	p->drc_index = drc_index;  in papr_scm_probe()
1451 	p->block_size = block_size;  in papr_scm_probe()
1452 	p->blocks = blocks;  in papr_scm_probe()
1453 	p->is_volatile = !of_property_read_bool(dn, "ibm,cache-flush-required");  in papr_scm_probe()
1454 	p->hcall_flush_required = of_property_read_bool(dn, "ibm,hcall-flush-required");  in papr_scm_probe()
1457 				 &p->dirty_shutdown_counter))  in papr_scm_probe()
1458 		p->dirty_shutdown_counter = 0;  in papr_scm_probe()
1476 	p->nd_set.cookie1 = get_unaligned_le64(&uuid_raw[0]);  in papr_scm_probe()
1477 	p->nd_set.cookie2 = get_unaligned_le64(&uuid_raw[8]);  in papr_scm_probe()
1480 	p->metadata_size = metadata_size;  in papr_scm_probe()
1481 	p->pdev = pdev;  in papr_scm_probe()
1484 	rc = drc_pmem_bind(p);  in papr_scm_probe()
1488 		rc = drc_pmem_query_n_bind(p);  in papr_scm_probe()
1491 		dev_err(&p->pdev->dev, "bind err: %d\n", rc);  in papr_scm_probe()
1497 	p->res.start = p->bound_addr;  in papr_scm_probe()
1498 	p->res.end   = p->bound_addr + p->blocks * p->block_size - 1;  in papr_scm_probe()
1499 	p->res.name  = pdev->name;  in papr_scm_probe()
1500 	p->res.flags = IORESOURCE_MEM;  in papr_scm_probe()
1503 	stat_size = drc_pmem_query_stats(p, NULL, 0);  in papr_scm_probe()
1505 		p->stat_buffer_len = stat_size;  in papr_scm_probe()
1506 		dev_dbg(&p->pdev->dev, "Max perf-stat size %lu-bytes\n",  in papr_scm_probe()
1507 			p->stat_buffer_len);  in papr_scm_probe()
1510 	rc = papr_scm_nvdimm_init(p);  in papr_scm_probe()
1514 	platform_set_drvdata(pdev, p);  in papr_scm_probe()
1515 	papr_scm_pmu_register(p);  in papr_scm_probe()
1519 err2:	drc_pmem_unbind(p);  in papr_scm_probe()
1520 err:	kfree(p);  in papr_scm_probe()
1526 	struct papr_scm_priv *p = platform_get_drvdata(pdev);  in papr_scm_remove()  local
1529 	list_del(&p->region_list);  in papr_scm_remove()
1532 	nvdimm_bus_unregister(p->bus);  in papr_scm_remove()
1533 	drc_pmem_unbind(p);  in papr_scm_remove()
1539 	kfree(p->bus_desc.provider_name);  in papr_scm_remove()
1540 	kfree(p);  in papr_scm_remove()