15b497af4SThomas Gleixner /* SPDX-License-Identifier: GPL-2.0-only */ 24d88a97aSDan Williams /* 34d88a97aSDan Williams * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 44d88a97aSDan Williams */ 54d88a97aSDan Williams #ifndef __ND_H__ 64d88a97aSDan Williams #define __ND_H__ 71f7df6f8SDan Williams #include <linux/libnvdimm.h> 8200c79daSDan Williams #include <linux/badblocks.h> 9f0dc089cSDan Williams #include <linux/blkdev.h> 104d88a97aSDan Williams #include <linux/device.h> 114d88a97aSDan Williams #include <linux/mutex.h> 124d88a97aSDan Williams #include <linux/ndctl.h> 13bf9bccc1SDan Williams #include <linux/types.h> 1471999466SDan Williams #include <linux/nd.h> 154a826c83SDan Williams #include "label.h" 164d88a97aSDan Williams 178c2f7e86SDan Williams enum { 185212e11fSVishal Verma /* 195212e11fSVishal Verma * Limits the maximum number of block apertures a dimm can 205212e11fSVishal Verma * support and is an input to the geometry/on-disk-format of a 215212e11fSVishal Verma * BTT instance 225212e11fSVishal Verma */ 235212e11fSVishal Verma ND_MAX_LANES = 256, 24fcae6957SVishal Verma INT_LBASIZE_ALIGNMENT = 64, 253ae3d67bSVishal Verma NVDIMM_IO_ATOMIC = 1, 268c2f7e86SDan Williams }; 278c2f7e86SDan Williams 284d88a97aSDan Williams struct nvdimm_drvdata { 294d88a97aSDan Williams struct device *dev; 3002881768SDan Williams int nslabel_size; 314d88a97aSDan Williams struct nd_cmd_get_config_size nsarea; 324d88a97aSDan Williams void *data; 33*5af96835SDan Williams bool cxl; 344a826c83SDan Williams int ns_current, ns_next; 354a826c83SDan Williams struct resource dpa; 36bf9bccc1SDan Williams struct kref kref; 374d88a97aSDan Williams }; 384d88a97aSDan Williams 39b4366a82SDan Williams static inline const u8 *nsl_ref_name(struct nvdimm_drvdata *ndd, 40b4366a82SDan Williams struct nd_namespace_label *nd_label) 41b4366a82SDan Williams { 42*5af96835SDan Williams if (ndd->cxl) 43*5af96835SDan Williams return nd_label->cxl.name; 44*5af96835SDan Williams return nd_label->efi.name; 45b4366a82SDan Williams } 46b4366a82SDan Williams 47b4366a82SDan Williams static inline u8 *nsl_get_name(struct nvdimm_drvdata *ndd, 48b4366a82SDan Williams struct nd_namespace_label *nd_label, u8 *name) 49b4366a82SDan Williams { 50*5af96835SDan Williams if (ndd->cxl) 51*5af96835SDan Williams return memcpy(name, nd_label->cxl.name, NSLABEL_NAME_LEN); 52*5af96835SDan Williams return memcpy(name, nd_label->efi.name, NSLABEL_NAME_LEN); 53b4366a82SDan Williams } 54b4366a82SDan Williams 558176f147SDan Williams static inline u8 *nsl_set_name(struct nvdimm_drvdata *ndd, 568176f147SDan Williams struct nd_namespace_label *nd_label, u8 *name) 578176f147SDan Williams { 588176f147SDan Williams if (!name) 598176f147SDan Williams return NULL; 60*5af96835SDan Williams if (ndd->cxl) 61*5af96835SDan Williams return memcpy(nd_label->cxl.name, name, NSLABEL_NAME_LEN); 62*5af96835SDan Williams return memcpy(nd_label->efi.name, name, NSLABEL_NAME_LEN); 638176f147SDan Williams } 648176f147SDan Williams 65b4366a82SDan Williams static inline u32 nsl_get_slot(struct nvdimm_drvdata *ndd, 66b4366a82SDan Williams struct nd_namespace_label *nd_label) 67b4366a82SDan Williams { 68*5af96835SDan Williams if (ndd->cxl) 69*5af96835SDan Williams return __le32_to_cpu(nd_label->cxl.slot); 70*5af96835SDan Williams return __le32_to_cpu(nd_label->efi.slot); 71b4366a82SDan Williams } 72b4366a82SDan Williams 738176f147SDan Williams static inline void nsl_set_slot(struct nvdimm_drvdata *ndd, 748176f147SDan Williams struct nd_namespace_label *nd_label, u32 slot) 758176f147SDan Williams { 76*5af96835SDan Williams if (ndd->cxl) 77*5af96835SDan Williams nd_label->cxl.slot = __cpu_to_le32(slot); 78*5af96835SDan Williams else 79*5af96835SDan Williams nd_label->efi.slot = __cpu_to_le32(slot); 808176f147SDan Williams } 818176f147SDan Williams 82b4366a82SDan Williams static inline u64 nsl_get_checksum(struct nvdimm_drvdata *ndd, 83b4366a82SDan Williams struct nd_namespace_label *nd_label) 84b4366a82SDan Williams { 85*5af96835SDan Williams if (ndd->cxl) 86*5af96835SDan Williams return __le64_to_cpu(nd_label->cxl.checksum); 87*5af96835SDan Williams return __le64_to_cpu(nd_label->efi.checksum); 88b4366a82SDan Williams } 89b4366a82SDan Williams 908176f147SDan Williams static inline void nsl_set_checksum(struct nvdimm_drvdata *ndd, 918176f147SDan Williams struct nd_namespace_label *nd_label, 928176f147SDan Williams u64 checksum) 938176f147SDan Williams { 94*5af96835SDan Williams if (ndd->cxl) 95*5af96835SDan Williams nd_label->cxl.checksum = __cpu_to_le64(checksum); 96*5af96835SDan Williams else 97*5af96835SDan Williams nd_label->efi.checksum = __cpu_to_le64(checksum); 988176f147SDan Williams } 998176f147SDan Williams 100b4366a82SDan Williams static inline u32 nsl_get_flags(struct nvdimm_drvdata *ndd, 101b4366a82SDan Williams struct nd_namespace_label *nd_label) 102b4366a82SDan Williams { 103*5af96835SDan Williams if (ndd->cxl) 104*5af96835SDan Williams return __le32_to_cpu(nd_label->cxl.flags); 105*5af96835SDan Williams return __le32_to_cpu(nd_label->efi.flags); 106b4366a82SDan Williams } 107b4366a82SDan Williams 1088176f147SDan Williams static inline void nsl_set_flags(struct nvdimm_drvdata *ndd, 1098176f147SDan Williams struct nd_namespace_label *nd_label, u32 flags) 1108176f147SDan Williams { 111*5af96835SDan Williams if (ndd->cxl) 112*5af96835SDan Williams nd_label->cxl.flags = __cpu_to_le32(flags); 113*5af96835SDan Williams else 114*5af96835SDan Williams nd_label->efi.flags = __cpu_to_le32(flags); 1158176f147SDan Williams } 1168176f147SDan Williams 117b4366a82SDan Williams static inline u64 nsl_get_dpa(struct nvdimm_drvdata *ndd, 118b4366a82SDan Williams struct nd_namespace_label *nd_label) 119b4366a82SDan Williams { 120*5af96835SDan Williams if (ndd->cxl) 121*5af96835SDan Williams return __le64_to_cpu(nd_label->cxl.dpa); 122*5af96835SDan Williams return __le64_to_cpu(nd_label->efi.dpa); 123b4366a82SDan Williams } 124b4366a82SDan Williams 1258176f147SDan Williams static inline void nsl_set_dpa(struct nvdimm_drvdata *ndd, 1268176f147SDan Williams struct nd_namespace_label *nd_label, u64 dpa) 1278176f147SDan Williams { 128*5af96835SDan Williams if (ndd->cxl) 129*5af96835SDan Williams nd_label->cxl.dpa = __cpu_to_le64(dpa); 130*5af96835SDan Williams else 131*5af96835SDan Williams nd_label->efi.dpa = __cpu_to_le64(dpa); 1328176f147SDan Williams } 1338176f147SDan Williams 134b4366a82SDan Williams static inline u64 nsl_get_rawsize(struct nvdimm_drvdata *ndd, 135b4366a82SDan Williams struct nd_namespace_label *nd_label) 136b4366a82SDan Williams { 137*5af96835SDan Williams if (ndd->cxl) 138*5af96835SDan Williams return __le64_to_cpu(nd_label->cxl.rawsize); 139*5af96835SDan Williams return __le64_to_cpu(nd_label->efi.rawsize); 140b4366a82SDan Williams } 141b4366a82SDan Williams 1428176f147SDan Williams static inline void nsl_set_rawsize(struct nvdimm_drvdata *ndd, 1438176f147SDan Williams struct nd_namespace_label *nd_label, 1448176f147SDan Williams u64 rawsize) 1458176f147SDan Williams { 146*5af96835SDan Williams if (ndd->cxl) 147*5af96835SDan Williams nd_label->cxl.rawsize = __cpu_to_le64(rawsize); 148*5af96835SDan Williams else 149*5af96835SDan Williams nd_label->efi.rawsize = __cpu_to_le64(rawsize); 1508176f147SDan Williams } 1518176f147SDan Williams 152b4366a82SDan Williams static inline u64 nsl_get_isetcookie(struct nvdimm_drvdata *ndd, 153b4366a82SDan Williams struct nd_namespace_label *nd_label) 154b4366a82SDan Williams { 155*5af96835SDan Williams /* WARN future refactor attempts that break this assumption */ 156*5af96835SDan Williams if (dev_WARN_ONCE(ndd->dev, ndd->cxl, 157*5af96835SDan Williams "CXL labels do not use the isetcookie concept\n")) 158*5af96835SDan Williams return 0; 159*5af96835SDan Williams return __le64_to_cpu(nd_label->efi.isetcookie); 160b4366a82SDan Williams } 161b4366a82SDan Williams 1628176f147SDan Williams static inline void nsl_set_isetcookie(struct nvdimm_drvdata *ndd, 1638176f147SDan Williams struct nd_namespace_label *nd_label, 1648176f147SDan Williams u64 isetcookie) 1658176f147SDan Williams { 166*5af96835SDan Williams if (!ndd->cxl) 167*5af96835SDan Williams nd_label->efi.isetcookie = __cpu_to_le64(isetcookie); 1688176f147SDan Williams } 1698176f147SDan Williams 1709761b02dSDan Williams static inline bool nsl_validate_isetcookie(struct nvdimm_drvdata *ndd, 1719761b02dSDan Williams struct nd_namespace_label *nd_label, 1729761b02dSDan Williams u64 cookie) 1739761b02dSDan Williams { 174*5af96835SDan Williams /* 175*5af96835SDan Williams * Let the EFI and CXL validation comingle, where fields that 176*5af96835SDan Williams * don't matter to CXL always validate. 177*5af96835SDan Williams */ 178*5af96835SDan Williams if (ndd->cxl) 179*5af96835SDan Williams return true; 180*5af96835SDan Williams return cookie == __le64_to_cpu(nd_label->efi.isetcookie); 1819761b02dSDan Williams } 1829761b02dSDan Williams 183b4366a82SDan Williams static inline u16 nsl_get_position(struct nvdimm_drvdata *ndd, 184b4366a82SDan Williams struct nd_namespace_label *nd_label) 185b4366a82SDan Williams { 186*5af96835SDan Williams if (ndd->cxl) 187*5af96835SDan Williams return __le16_to_cpu(nd_label->cxl.position); 188*5af96835SDan Williams return __le16_to_cpu(nd_label->efi.position); 189b4366a82SDan Williams } 190b4366a82SDan Williams 1918176f147SDan Williams static inline void nsl_set_position(struct nvdimm_drvdata *ndd, 1928176f147SDan Williams struct nd_namespace_label *nd_label, 1938176f147SDan Williams u16 position) 1948176f147SDan Williams { 195*5af96835SDan Williams if (ndd->cxl) 196*5af96835SDan Williams nd_label->cxl.position = __cpu_to_le16(position); 197*5af96835SDan Williams else 198*5af96835SDan Williams nd_label->efi.position = __cpu_to_le16(position); 1998176f147SDan Williams } 2008176f147SDan Williams 201b4366a82SDan Williams static inline u16 nsl_get_nlabel(struct nvdimm_drvdata *ndd, 202b4366a82SDan Williams struct nd_namespace_label *nd_label) 203b4366a82SDan Williams { 204*5af96835SDan Williams if (ndd->cxl) 205*5af96835SDan Williams return 0; 206*5af96835SDan Williams return __le16_to_cpu(nd_label->efi.nlabel); 207b4366a82SDan Williams } 208b4366a82SDan Williams 2098176f147SDan Williams static inline void nsl_set_nlabel(struct nvdimm_drvdata *ndd, 2108176f147SDan Williams struct nd_namespace_label *nd_label, 2118176f147SDan Williams u16 nlabel) 2128176f147SDan Williams { 213*5af96835SDan Williams if (!ndd->cxl) 214*5af96835SDan Williams nd_label->efi.nlabel = __cpu_to_le16(nlabel); 2158176f147SDan Williams } 2168176f147SDan Williams 21742e192aaSDan Williams static inline u16 nsl_get_nrange(struct nvdimm_drvdata *ndd, 21842e192aaSDan Williams struct nd_namespace_label *nd_label) 21942e192aaSDan Williams { 220*5af96835SDan Williams if (ndd->cxl) 221*5af96835SDan Williams return __le16_to_cpu(nd_label->cxl.nrange); 22242e192aaSDan Williams return 1; 22342e192aaSDan Williams } 22442e192aaSDan Williams 22542e192aaSDan Williams static inline void nsl_set_nrange(struct nvdimm_drvdata *ndd, 22642e192aaSDan Williams struct nd_namespace_label *nd_label, 22742e192aaSDan Williams u16 nrange) 22842e192aaSDan Williams { 229*5af96835SDan Williams if (ndd->cxl) 230*5af96835SDan Williams nd_label->cxl.nrange = __cpu_to_le16(nrange); 23142e192aaSDan Williams } 23242e192aaSDan Williams 233b4366a82SDan Williams static inline u64 nsl_get_lbasize(struct nvdimm_drvdata *ndd, 234b4366a82SDan Williams struct nd_namespace_label *nd_label) 235b4366a82SDan Williams { 236*5af96835SDan Williams /* 237*5af96835SDan Williams * Yes, for some reason the EFI labels convey a massive 64-bit 238*5af96835SDan Williams * lbasize, that got fixed for CXL. 239*5af96835SDan Williams */ 240*5af96835SDan Williams if (ndd->cxl) 241*5af96835SDan Williams return __le16_to_cpu(nd_label->cxl.lbasize); 242*5af96835SDan Williams return __le64_to_cpu(nd_label->efi.lbasize); 243b4366a82SDan Williams } 244b4366a82SDan Williams 2458176f147SDan Williams static inline void nsl_set_lbasize(struct nvdimm_drvdata *ndd, 2468176f147SDan Williams struct nd_namespace_label *nd_label, 2478176f147SDan Williams u64 lbasize) 2488176f147SDan Williams { 249*5af96835SDan Williams if (ndd->cxl) 250*5af96835SDan Williams nd_label->cxl.lbasize = __cpu_to_le16(lbasize); 251*5af96835SDan Williams else 252*5af96835SDan Williams nd_label->efi.lbasize = __cpu_to_le64(lbasize); 2538176f147SDan Williams } 2548176f147SDan Williams 255d1c6e08eSDan Williams static inline const uuid_t *nsl_get_uuid(struct nvdimm_drvdata *ndd, 256d1c6e08eSDan Williams struct nd_namespace_label *nd_label, 257d1c6e08eSDan Williams uuid_t *uuid) 258d1c6e08eSDan Williams { 259*5af96835SDan Williams if (ndd->cxl) 260*5af96835SDan Williams import_uuid(uuid, nd_label->cxl.uuid); 261*5af96835SDan Williams else 262*5af96835SDan Williams import_uuid(uuid, nd_label->efi.uuid); 263d1c6e08eSDan Williams return uuid; 264d1c6e08eSDan Williams } 265d1c6e08eSDan Williams 266d1c6e08eSDan Williams static inline const uuid_t *nsl_set_uuid(struct nvdimm_drvdata *ndd, 267d1c6e08eSDan Williams struct nd_namespace_label *nd_label, 268d1c6e08eSDan Williams const uuid_t *uuid) 269d1c6e08eSDan Williams { 270*5af96835SDan Williams if (ndd->cxl) 271*5af96835SDan Williams export_uuid(nd_label->cxl.uuid, uuid); 272*5af96835SDan Williams else 273*5af96835SDan Williams export_uuid(nd_label->efi.uuid, uuid); 274d1c6e08eSDan Williams return uuid; 275d1c6e08eSDan Williams } 276d1c6e08eSDan Williams 277d1c6e08eSDan Williams static inline bool nsl_uuid_equal(struct nvdimm_drvdata *ndd, 278d1c6e08eSDan Williams struct nd_namespace_label *nd_label, 279d1c6e08eSDan Williams const uuid_t *uuid) 280d1c6e08eSDan Williams { 281d1c6e08eSDan Williams uuid_t tmp; 282d1c6e08eSDan Williams 283*5af96835SDan Williams if (ndd->cxl) 284*5af96835SDan Williams import_uuid(&tmp, nd_label->cxl.uuid); 285*5af96835SDan Williams else 286*5af96835SDan Williams import_uuid(&tmp, nd_label->efi.uuid); 287d1c6e08eSDan Williams return uuid_equal(&tmp, uuid); 288d1c6e08eSDan Williams } 289d1c6e08eSDan Williams 290d1c6e08eSDan Williams static inline const u8 *nsl_uuid_raw(struct nvdimm_drvdata *ndd, 291d1c6e08eSDan Williams struct nd_namespace_label *nd_label) 292d1c6e08eSDan Williams { 293*5af96835SDan Williams if (ndd->cxl) 294*5af96835SDan Williams return nd_label->cxl.uuid; 295*5af96835SDan Williams return nd_label->efi.uuid; 296d1c6e08eSDan Williams } 297d1c6e08eSDan Williams 298f56541a7SDan Williams bool nsl_validate_blk_isetcookie(struct nvdimm_drvdata *ndd, 299f56541a7SDan Williams struct nd_namespace_label *nd_label, 300f56541a7SDan Williams u64 isetcookie); 3018b03aa0eSDan Williams bool nsl_validate_type_guid(struct nvdimm_drvdata *ndd, 3028b03aa0eSDan Williams struct nd_namespace_label *nd_label, guid_t *guid); 303a6e6d722SDan Williams enum nvdimm_claim_class nsl_get_claim_class(struct nvdimm_drvdata *ndd, 304a6e6d722SDan Williams struct nd_namespace_label *nd_label); 305f56541a7SDan Williams 306e5ae3b25SDan Williams struct nd_region_data { 307e5ae3b25SDan Williams int ns_count; 308e5ae3b25SDan Williams int ns_active; 309595c7307SDan Williams unsigned int hints_shift; 3109106137cSGustavo A. R. Silva void __iomem *flush_wpq[]; 3113d88002eSDan Williams }; 3123d88002eSDan Williams 313595c7307SDan Williams static inline void __iomem *ndrd_get_flush_wpq(struct nd_region_data *ndrd, 314595c7307SDan Williams int dimm, int hint) 315595c7307SDan Williams { 316595c7307SDan Williams unsigned int num = 1 << ndrd->hints_shift; 317595c7307SDan Williams unsigned int mask = num - 1; 318595c7307SDan Williams 319595c7307SDan Williams return ndrd->flush_wpq[dimm * num + (hint & mask)]; 320595c7307SDan Williams } 321595c7307SDan Williams 322595c7307SDan Williams static inline void ndrd_set_flush_wpq(struct nd_region_data *ndrd, int dimm, 323595c7307SDan Williams int hint, void __iomem *flush) 324595c7307SDan Williams { 325595c7307SDan Williams unsigned int num = 1 << ndrd->hints_shift; 326595c7307SDan Williams unsigned int mask = num - 1; 327595c7307SDan Williams 328595c7307SDan Williams ndrd->flush_wpq[dimm * num + (hint & mask)] = flush; 329595c7307SDan Williams } 330595c7307SDan Williams 3314a826c83SDan Williams static inline struct nd_namespace_index *to_namespace_index( 3324a826c83SDan Williams struct nvdimm_drvdata *ndd, int i) 3334a826c83SDan Williams { 3344a826c83SDan Williams if (i < 0) 3354a826c83SDan Williams return NULL; 3364a826c83SDan Williams 3374a826c83SDan Williams return ndd->data + sizeof_namespace_index(ndd) * i; 3384a826c83SDan Williams } 3394a826c83SDan Williams 3404a826c83SDan Williams static inline struct nd_namespace_index *to_current_namespace_index( 3414a826c83SDan Williams struct nvdimm_drvdata *ndd) 3424a826c83SDan Williams { 3434a826c83SDan Williams return to_namespace_index(ndd, ndd->ns_current); 3444a826c83SDan Williams } 3454a826c83SDan Williams 3464a826c83SDan Williams static inline struct nd_namespace_index *to_next_namespace_index( 3474a826c83SDan Williams struct nvdimm_drvdata *ndd) 3484a826c83SDan Williams { 3494a826c83SDan Williams return to_namespace_index(ndd, ndd->ns_next); 3504a826c83SDan Williams } 3514a826c83SDan Williams 352564e871aSDan Williams unsigned sizeof_namespace_label(struct nvdimm_drvdata *ndd); 353564e871aSDan Williams 354*5af96835SDan Williams #define efi_namespace_label_has(ndd, field) \ 355*5af96835SDan Williams (!ndd->cxl && offsetof(struct nvdimm_efi_label, field) \ 356564e871aSDan Williams < sizeof_namespace_label(ndd)) 357564e871aSDan Williams 3584a826c83SDan Williams #define nd_dbg_dpa(r, d, res, fmt, arg...) \ 3594a826c83SDan Williams dev_dbg((r) ? &(r)->dev : (d)->dev, "%s: %.13s: %#llx @ %#llx " fmt, \ 3604a826c83SDan Williams (r) ? dev_name((d)->dev) : "", res ? res->name : "null", \ 3614a826c83SDan Williams (unsigned long long) (res ? resource_size(res) : 0), \ 3624a826c83SDan Williams (unsigned long long) (res ? res->start : 0), ##arg) 3634a826c83SDan Williams 364bf9bccc1SDan Williams #define for_each_dpa_resource(ndd, res) \ 365bf9bccc1SDan Williams for (res = (ndd)->dpa.child; res; res = res->sibling) 366bf9bccc1SDan Williams 3674a826c83SDan Williams #define for_each_dpa_resource_safe(ndd, res, next) \ 3684a826c83SDan Williams for (res = (ndd)->dpa.child, next = res ? res->sibling : NULL; \ 3694a826c83SDan Williams res; res = next, next = next ? next->sibling : NULL) 3704a826c83SDan Williams 3715212e11fSVishal Verma struct nd_percpu_lane { 3725212e11fSVishal Verma int count; 3735212e11fSVishal Verma spinlock_t lock; 3745212e11fSVishal Verma }; 3755212e11fSVishal Verma 376c4703ce1SDan Williams enum nd_label_flags { 377c4703ce1SDan Williams ND_LABEL_REAP, 378c4703ce1SDan Williams }; 379ae8219f1SDan Williams struct nd_label_ent { 380ae8219f1SDan Williams struct list_head list; 381c4703ce1SDan Williams unsigned long flags; 382ae8219f1SDan Williams struct nd_namespace_label *label; 383ae8219f1SDan Williams }; 384ae8219f1SDan Williams 385ae8219f1SDan Williams enum nd_mapping_lock_class { 386ae8219f1SDan Williams ND_MAPPING_CLASS0, 387ae8219f1SDan Williams ND_MAPPING_UUID_SCAN, 388ae8219f1SDan Williams }; 389ae8219f1SDan Williams 39044c462ebSDan Williams struct nd_mapping { 39144c462ebSDan Williams struct nvdimm *nvdimm; 39244c462ebSDan Williams u64 start; 39344c462ebSDan Williams u64 size; 394401c0a19SDan Williams int position; 395ae8219f1SDan Williams struct list_head labels; 396ae8219f1SDan Williams struct mutex lock; 39744c462ebSDan Williams /* 39844c462ebSDan Williams * @ndd is for private use at region enable / disable time for 39944c462ebSDan Williams * get_ndd() + put_ndd(), all other nd_mapping to ndd 40044c462ebSDan Williams * conversions use to_ndd() which respects enabled state of the 40144c462ebSDan Williams * nvdimm. 40244c462ebSDan Williams */ 40344c462ebSDan Williams struct nvdimm_drvdata *ndd; 40444c462ebSDan Williams }; 40544c462ebSDan Williams 4061f7df6f8SDan Williams struct nd_region { 4071f7df6f8SDan Williams struct device dev; 4081b40e09aSDan Williams struct ida ns_ida; 4098c2f7e86SDan Williams struct ida btt_ida; 410e1455744SDan Williams struct ida pfn_ida; 411cd03412aSDan Williams struct ida dax_ida; 412004f1afbSDan Williams unsigned long flags; 413bf9bccc1SDan Williams struct device *ns_seed; 4148c2f7e86SDan Williams struct device *btt_seed; 415e1455744SDan Williams struct device *pfn_seed; 416cd03412aSDan Williams struct device *dax_seed; 4172522afb8SDan Williams unsigned long align; 4181f7df6f8SDan Williams u16 ndr_mappings; 4191f7df6f8SDan Williams u64 ndr_size; 4201f7df6f8SDan Williams u64 ndr_start; 4218fc5c735SDan Williams int id, num_lanes, ro, numa_node, target_node; 4221f7df6f8SDan Williams void *provider_data; 423975750a9SToshi Kani struct kernfs_node *bb_state; 4246a6bef90SDave Jiang struct badblocks bb; 425eaf96153SDan Williams struct nd_interleave_set *nd_set; 4265212e11fSVishal Verma struct nd_percpu_lane __percpu *lane; 427c5d4355dSPankaj Gupta int (*flush)(struct nd_region *nd_region, struct bio *bio); 4289106137cSGustavo A. R. Silva struct nd_mapping mapping[]; 4291f7df6f8SDan Williams }; 4301f7df6f8SDan Williams 4318172db92SDan Williams static inline bool nsl_validate_nlabel(struct nd_region *nd_region, 4328172db92SDan Williams struct nvdimm_drvdata *ndd, 4338172db92SDan Williams struct nd_namespace_label *nd_label) 4348172db92SDan Williams { 435*5af96835SDan Williams if (ndd->cxl) 436*5af96835SDan Williams return true; 4378172db92SDan Williams return nsl_get_nlabel(ndd, nd_label) == nd_region->ndr_mappings; 4388172db92SDan Williams } 4398172db92SDan Williams 440047fc8a1SRoss Zwisler struct nd_blk_region { 441047fc8a1SRoss Zwisler int (*enable)(struct nvdimm_bus *nvdimm_bus, struct device *dev); 442047fc8a1SRoss Zwisler int (*do_io)(struct nd_blk_region *ndbr, resource_size_t dpa, 443047fc8a1SRoss Zwisler void *iobuf, u64 len, int rw); 444047fc8a1SRoss Zwisler void *blk_provider_data; 445047fc8a1SRoss Zwisler struct nd_region nd_region; 446047fc8a1SRoss Zwisler }; 447047fc8a1SRoss Zwisler 4484a826c83SDan Williams /* 4494a826c83SDan Williams * Lookup next in the repeating sequence of 01, 10, and 11. 4504a826c83SDan Williams */ 4514a826c83SDan Williams static inline unsigned nd_inc_seq(unsigned seq) 4524a826c83SDan Williams { 4534a826c83SDan Williams static const unsigned next[] = { 0, 2, 3, 1 }; 4544a826c83SDan Williams 4554a826c83SDan Williams return next[seq & 3]; 4564a826c83SDan Williams } 457f524bf27SDan Williams 4585212e11fSVishal Verma struct btt; 4598c2f7e86SDan Williams struct nd_btt { 4608c2f7e86SDan Williams struct device dev; 4618c2f7e86SDan Williams struct nd_namespace_common *ndns; 4625212e11fSVishal Verma struct btt *btt; 4638c2f7e86SDan Williams unsigned long lbasize; 464abe8b4e3SVishal Verma u64 size; 465d1c6e08eSDan Williams uuid_t *uuid; 4668c2f7e86SDan Williams int id; 46714e49454SVishal Verma int initial_offset; 46814e49454SVishal Verma u16 version_major; 46914e49454SVishal Verma u16 version_minor; 4708c2f7e86SDan Williams }; 4718c2f7e86SDan Williams 472e1455744SDan Williams enum nd_pfn_mode { 473e1455744SDan Williams PFN_MODE_NONE, 474e1455744SDan Williams PFN_MODE_RAM, 475e1455744SDan Williams PFN_MODE_PMEM, 476e1455744SDan Williams }; 477e1455744SDan Williams 478e1455744SDan Williams struct nd_pfn { 479e1455744SDan Williams int id; 480d1c6e08eSDan Williams uuid_t *uuid; 481e1455744SDan Williams struct device dev; 482315c5625SDan Williams unsigned long align; 483e1455744SDan Williams unsigned long npfns; 484e1455744SDan Williams enum nd_pfn_mode mode; 485e1455744SDan Williams struct nd_pfn_sb *pfn_sb; 486e1455744SDan Williams struct nd_namespace_common *ndns; 487e1455744SDan Williams }; 488e1455744SDan Williams 489cd03412aSDan Williams struct nd_dax { 490cd03412aSDan Williams struct nd_pfn nd_pfn; 491cd03412aSDan Williams }; 492cd03412aSDan Williams 4938f4b01fcSAneesh Kumar K.V static inline u32 nd_info_block_reserve(void) 4948f4b01fcSAneesh Kumar K.V { 4958f4b01fcSAneesh Kumar K.V return ALIGN(SZ_8K, PAGE_SIZE); 4968f4b01fcSAneesh Kumar K.V } 4978f4b01fcSAneesh Kumar K.V 4984d88a97aSDan Williams enum nd_async_mode { 4994d88a97aSDan Williams ND_SYNC, 5004d88a97aSDan Williams ND_ASYNC, 5014d88a97aSDan Williams }; 5024d88a97aSDan Williams 50341cd8b70SVishal Verma int nd_integrity_init(struct gendisk *disk, unsigned long meta_size); 504bf9bccc1SDan Williams void wait_nvdimm_bus_probe_idle(struct device *dev); 5054d88a97aSDan Williams void nd_device_register(struct device *dev); 5064d88a97aSDan Williams void nd_device_unregister(struct device *dev, enum nd_async_mode mode); 50771999466SDan Williams void nd_device_notify(struct device *dev, enum nvdimm_event event); 508d1c6e08eSDan Williams int nd_uuid_store(struct device *dev, uuid_t **uuid_out, const char *buf, 509bf9bccc1SDan Williams size_t len); 510b2c48f9fSDan Williams ssize_t nd_size_select_show(unsigned long current_size, 5111b40e09aSDan Williams const unsigned long *supported, char *buf); 512b2c48f9fSDan Williams ssize_t nd_size_select_store(struct device *dev, const char *buf, 513b2c48f9fSDan Williams unsigned long *current_size, const unsigned long *supported); 5144d88a97aSDan Williams int __init nvdimm_init(void); 5153d88002eSDan Williams int __init nd_region_init(void); 516b3fde74eSDan Williams int __init nd_label_init(void); 5174d88a97aSDan Williams void nvdimm_exit(void); 5183d88002eSDan Williams void nd_region_exit(void); 519bf9bccc1SDan Williams struct nvdimm; 520adbb6829SDan Williams extern const struct attribute_group nd_device_attribute_group; 521e2f6a0e3SDan Williams extern const struct attribute_group nd_numa_attribute_group; 522e755799aSDan Williams extern const struct attribute_group *nvdimm_bus_attribute_groups[]; 523bf9bccc1SDan Williams struct nvdimm_drvdata *to_ndd(struct nd_mapping *nd_mapping); 524aee65987SToshi Kani int nvdimm_check_config_data(struct device *dev); 5254d88a97aSDan Williams int nvdimm_init_nsarea(struct nvdimm_drvdata *ndd); 5264d88a97aSDan Williams int nvdimm_init_config_data(struct nvdimm_drvdata *ndd); 5272d657d17SAlexander Duyck int nvdimm_get_config_data(struct nvdimm_drvdata *ndd, void *buf, 5282d657d17SAlexander Duyck size_t offset, size_t len); 529f524bf27SDan Williams int nvdimm_set_config_data(struct nvdimm_drvdata *ndd, size_t offset, 530f524bf27SDan Williams void *buf, size_t len); 53159e64739SDan Williams long nvdimm_clear_poison(struct device *dev, phys_addr_t phys, 53259e64739SDan Williams unsigned int len); 533a0e37452SDan Williams void nvdimm_set_labeling(struct device *dev); 5348f078b38SDan Williams void nvdimm_set_locked(struct device *dev); 535d34cb808SDan Williams void nvdimm_clear_locked(struct device *dev); 5361cd73865SDan Williams int nvdimm_security_setup_events(struct device *dev); 5374c6926a2SDave Jiang #if IS_ENABLED(CONFIG_NVDIMM_KEYS) 5384c6926a2SDave Jiang int nvdimm_security_unlock(struct device *dev); 5394c6926a2SDave Jiang #else 5404c6926a2SDave Jiang static inline int nvdimm_security_unlock(struct device *dev) 5414c6926a2SDave Jiang { 5424c6926a2SDave Jiang return 0; 5434c6926a2SDave Jiang } 5444c6926a2SDave Jiang #endif 5458c2f7e86SDan Williams struct nd_btt *to_nd_btt(struct device *dev); 546e1455744SDan Williams 547e1455744SDan Williams struct nd_gen_sb { 548e1455744SDan Williams char reserved[SZ_4K - 8]; 549e1455744SDan Williams __le64 checksum; 550e1455744SDan Williams }; 551e1455744SDan Williams 552e1455744SDan Williams u64 nd_sb_checksum(struct nd_gen_sb *sb); 5538c2f7e86SDan Williams #if IS_ENABLED(CONFIG_BTT) 554200c79daSDan Williams int nd_btt_probe(struct device *dev, struct nd_namespace_common *ndns); 5558c2f7e86SDan Williams bool is_nd_btt(struct device *dev); 5568c2f7e86SDan Williams struct device *nd_btt_create(struct nd_region *nd_region); 5578c2f7e86SDan Williams #else 558e32bc729SDan Williams static inline int nd_btt_probe(struct device *dev, 559200c79daSDan Williams struct nd_namespace_common *ndns) 5608c2f7e86SDan Williams { 5618c2f7e86SDan Williams return -ENODEV; 5628c2f7e86SDan Williams } 5638c2f7e86SDan Williams 5648c2f7e86SDan Williams static inline bool is_nd_btt(struct device *dev) 5658c2f7e86SDan Williams { 5668c2f7e86SDan Williams return false; 5678c2f7e86SDan Williams } 5688c2f7e86SDan Williams 5698c2f7e86SDan Williams static inline struct device *nd_btt_create(struct nd_region *nd_region) 5708c2f7e86SDan Williams { 5718c2f7e86SDan Williams return NULL; 5728c2f7e86SDan Williams } 5738c2f7e86SDan Williams #endif 574e1455744SDan Williams 575e1455744SDan Williams struct nd_pfn *to_nd_pfn(struct device *dev); 576e1455744SDan Williams #if IS_ENABLED(CONFIG_NVDIMM_PFN) 5770dd69643SOliver O'Halloran 578f5376699SAneesh Kumar K.V #define MAX_NVDIMM_ALIGN 4 5790dd69643SOliver O'Halloran 580200c79daSDan Williams int nd_pfn_probe(struct device *dev, struct nd_namespace_common *ndns); 581e1455744SDan Williams bool is_nd_pfn(struct device *dev); 582e1455744SDan Williams struct device *nd_pfn_create(struct nd_region *nd_region); 583cd03412aSDan Williams struct device *nd_pfn_devinit(struct nd_pfn *nd_pfn, 584cd03412aSDan Williams struct nd_namespace_common *ndns); 585c5ed9268SDan Williams int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig); 58678c81cc8SDan Williams extern const struct attribute_group *nd_pfn_attribute_groups[]; 587e1455744SDan Williams #else 588200c79daSDan Williams static inline int nd_pfn_probe(struct device *dev, 589200c79daSDan Williams struct nd_namespace_common *ndns) 590e1455744SDan Williams { 591e1455744SDan Williams return -ENODEV; 592e1455744SDan Williams } 593e1455744SDan Williams 594e1455744SDan Williams static inline bool is_nd_pfn(struct device *dev) 595e1455744SDan Williams { 596e1455744SDan Williams return false; 597e1455744SDan Williams } 598e1455744SDan Williams 599e1455744SDan Williams static inline struct device *nd_pfn_create(struct nd_region *nd_region) 600e1455744SDan Williams { 601e1455744SDan Williams return NULL; 602e1455744SDan Williams } 60332ab0a3fSDan Williams 604c5ed9268SDan Williams static inline int nd_pfn_validate(struct nd_pfn *nd_pfn, const char *sig) 60532ab0a3fSDan Williams { 60632ab0a3fSDan Williams return -ENODEV; 60732ab0a3fSDan Williams } 608e1455744SDan Williams #endif 609e1455744SDan Williams 610cd03412aSDan Williams struct nd_dax *to_nd_dax(struct device *dev); 611cd03412aSDan Williams #if IS_ENABLED(CONFIG_NVDIMM_DAX) 612c5ed9268SDan Williams int nd_dax_probe(struct device *dev, struct nd_namespace_common *ndns); 613cd03412aSDan Williams bool is_nd_dax(struct device *dev); 614cd03412aSDan Williams struct device *nd_dax_create(struct nd_region *nd_region); 615cd03412aSDan Williams #else 616c5ed9268SDan Williams static inline int nd_dax_probe(struct device *dev, 617c5ed9268SDan Williams struct nd_namespace_common *ndns) 618c5ed9268SDan Williams { 619c5ed9268SDan Williams return -ENODEV; 620c5ed9268SDan Williams } 621c5ed9268SDan Williams 622cd03412aSDan Williams static inline bool is_nd_dax(struct device *dev) 623cd03412aSDan Williams { 624cd03412aSDan Williams return false; 625cd03412aSDan Williams } 626cd03412aSDan Williams 627cd03412aSDan Williams static inline struct device *nd_dax_create(struct nd_region *nd_region) 628cd03412aSDan Williams { 629cd03412aSDan Williams return NULL; 630cd03412aSDan Williams } 631cd03412aSDan Williams #endif 632cd03412aSDan Williams 6333d88002eSDan Williams int nd_region_to_nstype(struct nd_region *nd_region); 6343d88002eSDan Williams int nd_region_register_namespaces(struct nd_region *nd_region, int *err); 635c12c48ceSDan Williams u64 nd_region_interleave_set_cookie(struct nd_region *nd_region, 636c12c48ceSDan Williams struct nd_namespace_index *nsindex); 63786ef58a4SDan Williams u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region); 6383d88002eSDan Williams void nvdimm_bus_lock(struct device *dev); 6393d88002eSDan Williams void nvdimm_bus_unlock(struct device *dev); 6403d88002eSDan Williams bool is_nvdimm_bus_locked(struct device *dev); 64132f61d67SChristoph Hellwig void nvdimm_check_and_set_ro(struct gendisk *disk); 642bf9bccc1SDan Williams void nvdimm_drvdata_release(struct kref *kref); 643bf9bccc1SDan Williams void put_ndd(struct nvdimm_drvdata *ndd); 6444a826c83SDan Williams int nd_label_reserve_dpa(struct nvdimm_drvdata *ndd); 6454a826c83SDan Williams void nvdimm_free_dpa(struct nvdimm_drvdata *ndd, struct resource *res); 6464a826c83SDan Williams struct resource *nvdimm_allocate_dpa(struct nvdimm_drvdata *ndd, 6474a826c83SDan Williams struct nd_label_id *label_id, resource_size_t start, 6484a826c83SDan Williams resource_size_t n); 6498c2f7e86SDan Williams resource_size_t nvdimm_namespace_capacity(struct nd_namespace_common *ndns); 65008e6b3c6SDan Williams bool nvdimm_namespace_locked(struct nd_namespace_common *ndns); 6518c2f7e86SDan Williams struct nd_namespace_common *nvdimm_namespace_common_probe(struct device *dev); 6525212e11fSVishal Verma int nvdimm_namespace_attach_btt(struct nd_namespace_common *ndns); 653298f2bc5SDan Williams int nvdimm_namespace_detach_btt(struct nd_btt *nd_btt); 6545212e11fSVishal Verma const char *nvdimm_namespace_disk_name(struct nd_namespace_common *ndns, 6555212e11fSVishal Verma char *name); 656f979b13cSDan Williams unsigned int pmem_sector_size(struct nd_namespace_common *ndns); 657a4574f63SDan Williams struct range; 658a3901802SDan Williams void nvdimm_badblocks_populate(struct nd_region *nd_region, 659a4574f63SDan Williams struct badblocks *bb, const struct range *range); 6608f4b01fcSAneesh Kumar K.V int devm_namespace_enable(struct device *dev, struct nd_namespace_common *ndns, 6618f4b01fcSAneesh Kumar K.V resource_size_t size); 6628f4b01fcSAneesh Kumar K.V void devm_namespace_disable(struct device *dev, 6638f4b01fcSAneesh Kumar K.V struct nd_namespace_common *ndns); 664200c79daSDan Williams #if IS_ENABLED(CONFIG_ND_CLAIM) 665e96f0bf2SAneesh Kumar K.V /* max struct page size independent of kernel config */ 666e96f0bf2SAneesh Kumar K.V #define MAX_STRUCT_PAGE_SIZE 64 667e8d51348SChristoph Hellwig int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap); 668200c79daSDan Williams #else 669e8d51348SChristoph Hellwig static inline int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, 670e8d51348SChristoph Hellwig struct dev_pagemap *pgmap) 671ac515c08SDan Williams { 672e8d51348SChristoph Hellwig return -ENXIO; 673ac515c08SDan Williams } 674200c79daSDan Williams #endif 675047fc8a1SRoss Zwisler int nd_blk_region_init(struct nd_region *nd_region); 676e5ae3b25SDan Williams int nd_region_activate(struct nd_region *nd_region); 677200c79daSDan Williams static inline bool is_bad_pmem(struct badblocks *bb, sector_t sector, 678200c79daSDan Williams unsigned int len) 679200c79daSDan Williams { 680200c79daSDan Williams if (bb->count) { 681200c79daSDan Williams sector_t first_bad; 682200c79daSDan Williams int num_bad; 683200c79daSDan Williams 684200c79daSDan Williams return !!badblocks_check(bb, sector, len / 512, &first_bad, 685200c79daSDan Williams &num_bad); 686200c79daSDan Williams } 687200c79daSDan Williams 688200c79daSDan Williams return false; 689200c79daSDan Williams } 690047fc8a1SRoss Zwisler resource_size_t nd_namespace_blk_validate(struct nd_namespace_blk *nsblk); 691d1c6e08eSDan Williams const uuid_t *nd_dev_to_uuid(struct device *dev); 692004f1afbSDan Williams bool pmem_should_map_pages(struct device *dev); 6934d88a97aSDan Williams #endif /* __ND_H__ */ 694