1779dd20cSBen Widawsky // SPDX-License-Identifier: GPL-2.0-only
2779dd20cSBen Widawsky /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3779dd20cSBen Widawsky #include <linux/memregion.h>
4779dd20cSBen Widawsky #include <linux/genalloc.h>
5779dd20cSBen Widawsky #include <linux/device.h>
6779dd20cSBen Widawsky #include <linux/module.h>
7779dd20cSBen Widawsky #include <linux/slab.h>
8dd5ba0ebSBen Widawsky #include <linux/uuid.h>
9a32320b7SDan Williams #include <linux/sort.h>
10779dd20cSBen Widawsky #include <linux/idr.h>
1180d10a6cSBen Widawsky #include <cxlmem.h>
12779dd20cSBen Widawsky #include <cxl.h>
13779dd20cSBen Widawsky #include "core.h"
14779dd20cSBen Widawsky
15779dd20cSBen Widawsky /**
16779dd20cSBen Widawsky * DOC: cxl core region
17779dd20cSBen Widawsky *
18779dd20cSBen Widawsky * CXL Regions represent mapped memory capacity in system physical address
19779dd20cSBen Widawsky * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
20779dd20cSBen Widawsky * Memory ranges, Regions represent the active mapped capacity by the HDM
21779dd20cSBen Widawsky * Decoder Capability structures throughout the Host Bridges, Switches, and
22779dd20cSBen Widawsky * Endpoints in the topology.
23dd5ba0ebSBen Widawsky *
24dd5ba0ebSBen Widawsky * Region configuration has ordering constraints. UUID may be set at any time
25dd5ba0ebSBen Widawsky * but is only visible for persistent regions.
2680d10a6cSBen Widawsky * 1. Interleave granularity
2780d10a6cSBen Widawsky * 2. Interleave size
28b9686e8cSDan Williams * 3. Decoder targets
29779dd20cSBen Widawsky */
30779dd20cSBen Widawsky
31779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev);
32779dd20cSBen Widawsky
uuid_show(struct device * dev,struct device_attribute * attr,char * buf)33dd5ba0ebSBen Widawsky static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
34dd5ba0ebSBen Widawsky char *buf)
35dd5ba0ebSBen Widawsky {
36dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev);
37dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params;
38dd5ba0ebSBen Widawsky ssize_t rc;
39dd5ba0ebSBen Widawsky
40dd5ba0ebSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem);
41dd5ba0ebSBen Widawsky if (rc)
42dd5ba0ebSBen Widawsky return rc;
43a8e7d558SDan Williams if (cxlr->mode != CXL_DECODER_PMEM)
44a8e7d558SDan Williams rc = sysfs_emit(buf, "\n");
45a8e7d558SDan Williams else
46dd5ba0ebSBen Widawsky rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
47dd5ba0ebSBen Widawsky up_read(&cxl_region_rwsem);
48dd5ba0ebSBen Widawsky
49dd5ba0ebSBen Widawsky return rc;
50dd5ba0ebSBen Widawsky }
51dd5ba0ebSBen Widawsky
is_dup(struct device * match,void * data)52dd5ba0ebSBen Widawsky static int is_dup(struct device *match, void *data)
53dd5ba0ebSBen Widawsky {
54dd5ba0ebSBen Widawsky struct cxl_region_params *p;
55dd5ba0ebSBen Widawsky struct cxl_region *cxlr;
56dd5ba0ebSBen Widawsky uuid_t *uuid = data;
57dd5ba0ebSBen Widawsky
58dd5ba0ebSBen Widawsky if (!is_cxl_region(match))
59dd5ba0ebSBen Widawsky return 0;
60dd5ba0ebSBen Widawsky
61dd5ba0ebSBen Widawsky lockdep_assert_held(&cxl_region_rwsem);
62dd5ba0ebSBen Widawsky cxlr = to_cxl_region(match);
63dd5ba0ebSBen Widawsky p = &cxlr->params;
64dd5ba0ebSBen Widawsky
65dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, uuid)) {
66dd5ba0ebSBen Widawsky dev_dbg(match, "already has uuid: %pUb\n", uuid);
67dd5ba0ebSBen Widawsky return -EBUSY;
68dd5ba0ebSBen Widawsky }
69dd5ba0ebSBen Widawsky
70dd5ba0ebSBen Widawsky return 0;
71dd5ba0ebSBen Widawsky }
72dd5ba0ebSBen Widawsky
uuid_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)73dd5ba0ebSBen Widawsky static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
74dd5ba0ebSBen Widawsky const char *buf, size_t len)
75dd5ba0ebSBen Widawsky {
76dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev);
77dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params;
78dd5ba0ebSBen Widawsky uuid_t temp;
79dd5ba0ebSBen Widawsky ssize_t rc;
80dd5ba0ebSBen Widawsky
81dd5ba0ebSBen Widawsky if (len != UUID_STRING_LEN + 1)
82dd5ba0ebSBen Widawsky return -EINVAL;
83dd5ba0ebSBen Widawsky
84dd5ba0ebSBen Widawsky rc = uuid_parse(buf, &temp);
85dd5ba0ebSBen Widawsky if (rc)
86dd5ba0ebSBen Widawsky return rc;
87dd5ba0ebSBen Widawsky
88dd5ba0ebSBen Widawsky if (uuid_is_null(&temp))
89dd5ba0ebSBen Widawsky return -EINVAL;
90dd5ba0ebSBen Widawsky
91dd5ba0ebSBen Widawsky rc = down_write_killable(&cxl_region_rwsem);
92dd5ba0ebSBen Widawsky if (rc)
93dd5ba0ebSBen Widawsky return rc;
94dd5ba0ebSBen Widawsky
95dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, &temp))
96dd5ba0ebSBen Widawsky goto out;
97dd5ba0ebSBen Widawsky
98dd5ba0ebSBen Widawsky rc = -EBUSY;
99dd5ba0ebSBen Widawsky if (p->state >= CXL_CONFIG_ACTIVE)
100dd5ba0ebSBen Widawsky goto out;
101dd5ba0ebSBen Widawsky
102dd5ba0ebSBen Widawsky rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
103dd5ba0ebSBen Widawsky if (rc < 0)
104dd5ba0ebSBen Widawsky goto out;
105dd5ba0ebSBen Widawsky
106dd5ba0ebSBen Widawsky uuid_copy(&p->uuid, &temp);
107dd5ba0ebSBen Widawsky out:
108dd5ba0ebSBen Widawsky up_write(&cxl_region_rwsem);
109dd5ba0ebSBen Widawsky
110dd5ba0ebSBen Widawsky if (rc)
111dd5ba0ebSBen Widawsky return rc;
112dd5ba0ebSBen Widawsky return len;
113dd5ba0ebSBen Widawsky }
114dd5ba0ebSBen Widawsky static DEVICE_ATTR_RW(uuid);
115dd5ba0ebSBen Widawsky
cxl_rr_load(struct cxl_port * port,struct cxl_region * cxlr)116176baefbSDan Williams static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
117176baefbSDan Williams struct cxl_region *cxlr)
118176baefbSDan Williams {
119176baefbSDan Williams return xa_load(&port->regions, (unsigned long)cxlr);
120176baefbSDan Williams }
121176baefbSDan Williams
cxl_region_invalidate_memregion(struct cxl_region * cxlr)122d1257d09SDan Williams static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
123d1257d09SDan Williams {
124d1257d09SDan Williams if (!cpu_cache_has_invalidate_memregion()) {
125d1257d09SDan Williams if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
126d1257d09SDan Williams dev_warn_once(
127d1257d09SDan Williams &cxlr->dev,
128d1257d09SDan Williams "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
129d1257d09SDan Williams return 0;
130d1257d09SDan Williams } else {
131d1257d09SDan Williams dev_err(&cxlr->dev,
132d1257d09SDan Williams "Failed to synchronize CPU cache state\n");
133d1257d09SDan Williams return -ENXIO;
134d1257d09SDan Williams }
135d1257d09SDan Williams }
136d1257d09SDan Williams
137d1257d09SDan Williams cpu_cache_invalidate_memregion(IORES_DESC_CXL);
138d1257d09SDan Williams return 0;
139d1257d09SDan Williams }
140d1257d09SDan Williams
cxl_region_decode_reset(struct cxl_region * cxlr,int count)141176baefbSDan Williams static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
142176baefbSDan Williams {
143176baefbSDan Williams struct cxl_region_params *p = &cxlr->params;
144d1257d09SDan Williams int i, rc = 0;
145d1257d09SDan Williams
146d1257d09SDan Williams /*
147d1257d09SDan Williams * Before region teardown attempt to flush, and if the flush
148d1257d09SDan Williams * fails cancel the region teardown for data consistency
149d1257d09SDan Williams * concerns
150d1257d09SDan Williams */
151d1257d09SDan Williams rc = cxl_region_invalidate_memregion(cxlr);
152d1257d09SDan Williams if (rc)
153d1257d09SDan Williams return rc;
154176baefbSDan Williams
155176baefbSDan Williams for (i = count - 1; i >= 0; i--) {
156176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i];
157176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
158176baefbSDan Williams struct cxl_port *iter = cxled_to_port(cxled);
159030f8803SDan Williams struct cxl_dev_state *cxlds = cxlmd->cxlds;
160176baefbSDan Williams struct cxl_ep *ep;
161176baefbSDan Williams
162030f8803SDan Williams if (cxlds->rcd)
163030f8803SDan Williams goto endpoint_reset;
164030f8803SDan Williams
165176baefbSDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
166176baefbSDan Williams iter = to_cxl_port(iter->dev.parent);
167176baefbSDan Williams
168176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter;
169176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
170176baefbSDan Williams struct cxl_region_ref *cxl_rr;
171176baefbSDan Williams struct cxl_decoder *cxld;
172176baefbSDan Williams
173176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr);
174176baefbSDan Williams cxld = cxl_rr->decoder;
1754fa4302dSFan Ni if (cxld->reset)
176176baefbSDan Williams rc = cxld->reset(cxld);
177176baefbSDan Williams if (rc)
178176baefbSDan Williams return rc;
1792ab47045SDan Williams set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
180176baefbSDan Williams }
181176baefbSDan Williams
182030f8803SDan Williams endpoint_reset:
183176baefbSDan Williams rc = cxled->cxld.reset(&cxled->cxld);
184176baefbSDan Williams if (rc)
185176baefbSDan Williams return rc;
1862ab47045SDan Williams set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
187176baefbSDan Williams }
188176baefbSDan Williams
1892ab47045SDan Williams /* all decoders associated with this region have been torn down */
1902ab47045SDan Williams clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
1912ab47045SDan Williams
192176baefbSDan Williams return 0;
193176baefbSDan Williams }
194176baefbSDan Williams
commit_decoder(struct cxl_decoder * cxld)195af3ea9abSDan Williams static int commit_decoder(struct cxl_decoder *cxld)
196af3ea9abSDan Williams {
197af3ea9abSDan Williams struct cxl_switch_decoder *cxlsd = NULL;
198af3ea9abSDan Williams
199af3ea9abSDan Williams if (cxld->commit)
200af3ea9abSDan Williams return cxld->commit(cxld);
201af3ea9abSDan Williams
202af3ea9abSDan Williams if (is_switch_decoder(&cxld->dev))
203af3ea9abSDan Williams cxlsd = to_cxl_switch_decoder(&cxld->dev);
204af3ea9abSDan Williams
205af3ea9abSDan Williams if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
206af3ea9abSDan Williams "->commit() is required\n"))
207af3ea9abSDan Williams return -ENXIO;
208af3ea9abSDan Williams return 0;
209af3ea9abSDan Williams }
210af3ea9abSDan Williams
cxl_region_decode_commit(struct cxl_region * cxlr)211176baefbSDan Williams static int cxl_region_decode_commit(struct cxl_region *cxlr)
212176baefbSDan Williams {
213176baefbSDan Williams struct cxl_region_params *p = &cxlr->params;
21469c99613SDan Williams int i, rc = 0;
215176baefbSDan Williams
216176baefbSDan Williams for (i = 0; i < p->nr_targets; i++) {
217176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i];
218176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
219176baefbSDan Williams struct cxl_region_ref *cxl_rr;
220176baefbSDan Williams struct cxl_decoder *cxld;
221176baefbSDan Williams struct cxl_port *iter;
222176baefbSDan Williams struct cxl_ep *ep;
223176baefbSDan Williams
224176baefbSDan Williams /* commit bottom up */
225176baefbSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
226176baefbSDan Williams iter = to_cxl_port(iter->dev.parent)) {
227176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr);
228176baefbSDan Williams cxld = cxl_rr->decoder;
229af3ea9abSDan Williams rc = commit_decoder(cxld);
230176baefbSDan Williams if (rc)
231176baefbSDan Williams break;
232176baefbSDan Williams }
233176baefbSDan Williams
23469c99613SDan Williams if (rc) {
235176baefbSDan Williams /* programming @iter failed, teardown */
236176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
237176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
238176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr);
239176baefbSDan Williams cxld = cxl_rr->decoder;
2404fa4302dSFan Ni if (cxld->reset)
241176baefbSDan Williams cxld->reset(cxld);
242176baefbSDan Williams }
243176baefbSDan Williams
244176baefbSDan Williams cxled->cxld.reset(&cxled->cxld);
24569c99613SDan Williams goto err;
24669c99613SDan Williams }
247176baefbSDan Williams }
248176baefbSDan Williams
249176baefbSDan Williams return 0;
250176baefbSDan Williams
25169c99613SDan Williams err:
252176baefbSDan Williams /* undo the targets that were successfully committed */
253176baefbSDan Williams cxl_region_decode_reset(cxlr, i);
254176baefbSDan Williams return rc;
255176baefbSDan Williams }
256176baefbSDan Williams
commit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)257176baefbSDan Williams static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
258176baefbSDan Williams const char *buf, size_t len)
259176baefbSDan Williams {
260176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev);
261176baefbSDan Williams struct cxl_region_params *p = &cxlr->params;
262176baefbSDan Williams bool commit;
263176baefbSDan Williams ssize_t rc;
264176baefbSDan Williams
265176baefbSDan Williams rc = kstrtobool(buf, &commit);
266176baefbSDan Williams if (rc)
267176baefbSDan Williams return rc;
268176baefbSDan Williams
269176baefbSDan Williams rc = down_write_killable(&cxl_region_rwsem);
270176baefbSDan Williams if (rc)
271176baefbSDan Williams return rc;
272176baefbSDan Williams
273176baefbSDan Williams /* Already in the requested state? */
274176baefbSDan Williams if (commit && p->state >= CXL_CONFIG_COMMIT)
275176baefbSDan Williams goto out;
276176baefbSDan Williams if (!commit && p->state < CXL_CONFIG_COMMIT)
277176baefbSDan Williams goto out;
278176baefbSDan Williams
279176baefbSDan Williams /* Not ready to commit? */
280176baefbSDan Williams if (commit && p->state < CXL_CONFIG_ACTIVE) {
281176baefbSDan Williams rc = -ENXIO;
282176baefbSDan Williams goto out;
283176baefbSDan Williams }
284176baefbSDan Williams
285d1257d09SDan Williams /*
286d1257d09SDan Williams * Invalidate caches before region setup to drop any speculative
287d1257d09SDan Williams * consumption of this address space
288d1257d09SDan Williams */
289d1257d09SDan Williams rc = cxl_region_invalidate_memregion(cxlr);
290d1257d09SDan Williams if (rc)
29141f3c9f0SLi Zhijian goto out;
292d1257d09SDan Williams
293adfe1973SDan Williams if (commit) {
294176baefbSDan Williams rc = cxl_region_decode_commit(cxlr);
295adfe1973SDan Williams if (rc == 0)
296adfe1973SDan Williams p->state = CXL_CONFIG_COMMIT;
297adfe1973SDan Williams } else {
298176baefbSDan Williams p->state = CXL_CONFIG_RESET_PENDING;
299176baefbSDan Williams up_write(&cxl_region_rwsem);
300176baefbSDan Williams device_release_driver(&cxlr->dev);
301176baefbSDan Williams down_write(&cxl_region_rwsem);
302176baefbSDan Williams
303176baefbSDan Williams /*
304176baefbSDan Williams * The lock was dropped, so need to revalidate that the reset is
305176baefbSDan Williams * still pending.
306176baefbSDan Williams */
307adfe1973SDan Williams if (p->state == CXL_CONFIG_RESET_PENDING) {
308176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
309adfe1973SDan Williams /*
310adfe1973SDan Williams * Revert to committed since there may still be active
311adfe1973SDan Williams * decoders associated with this region, or move forward
312adfe1973SDan Williams * to active to mark the reset successful
313adfe1973SDan Williams */
314176baefbSDan Williams if (rc)
315176baefbSDan Williams p->state = CXL_CONFIG_COMMIT;
316adfe1973SDan Williams else
317176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE;
318adfe1973SDan Williams }
319adfe1973SDan Williams }
320176baefbSDan Williams
321176baefbSDan Williams out:
322176baefbSDan Williams up_write(&cxl_region_rwsem);
323176baefbSDan Williams
324176baefbSDan Williams if (rc)
325176baefbSDan Williams return rc;
326176baefbSDan Williams return len;
327176baefbSDan Williams }
328176baefbSDan Williams
commit_show(struct device * dev,struct device_attribute * attr,char * buf)329176baefbSDan Williams static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
330176baefbSDan Williams char *buf)
331176baefbSDan Williams {
332176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev);
333176baefbSDan Williams struct cxl_region_params *p = &cxlr->params;
334176baefbSDan Williams ssize_t rc;
335176baefbSDan Williams
336176baefbSDan Williams rc = down_read_interruptible(&cxl_region_rwsem);
337176baefbSDan Williams if (rc)
338176baefbSDan Williams return rc;
339176baefbSDan Williams rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
340176baefbSDan Williams up_read(&cxl_region_rwsem);
341176baefbSDan Williams
342176baefbSDan Williams return rc;
343176baefbSDan Williams }
344176baefbSDan Williams static DEVICE_ATTR_RW(commit);
345176baefbSDan Williams
cxl_region_visible(struct kobject * kobj,struct attribute * a,int n)346dd5ba0ebSBen Widawsky static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
347dd5ba0ebSBen Widawsky int n)
348dd5ba0ebSBen Widawsky {
349dd5ba0ebSBen Widawsky struct device *dev = kobj_to_dev(kobj);
350dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev);
351dd5ba0ebSBen Widawsky
352a8e7d558SDan Williams /*
353a8e7d558SDan Williams * Support tooling that expects to find a 'uuid' attribute for all
354a8e7d558SDan Williams * regions regardless of mode.
355a8e7d558SDan Williams */
356dd5ba0ebSBen Widawsky if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
357a8e7d558SDan Williams return 0444;
358dd5ba0ebSBen Widawsky return a->mode;
359dd5ba0ebSBen Widawsky }
360dd5ba0ebSBen Widawsky
interleave_ways_show(struct device * dev,struct device_attribute * attr,char * buf)36180d10a6cSBen Widawsky static ssize_t interleave_ways_show(struct device *dev,
36280d10a6cSBen Widawsky struct device_attribute *attr, char *buf)
36380d10a6cSBen Widawsky {
36480d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev);
36580d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params;
36680d10a6cSBen Widawsky ssize_t rc;
36780d10a6cSBen Widawsky
36880d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem);
36980d10a6cSBen Widawsky if (rc)
37080d10a6cSBen Widawsky return rc;
37180d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
37280d10a6cSBen Widawsky up_read(&cxl_region_rwsem);
37380d10a6cSBen Widawsky
37480d10a6cSBen Widawsky return rc;
37580d10a6cSBen Widawsky }
37680d10a6cSBen Widawsky
377b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void);
378b9686e8cSDan Williams
interleave_ways_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)37980d10a6cSBen Widawsky static ssize_t interleave_ways_store(struct device *dev,
38080d10a6cSBen Widawsky struct device_attribute *attr,
38180d10a6cSBen Widawsky const char *buf, size_t len)
38280d10a6cSBen Widawsky {
38380d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
38480d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
38580d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev);
38680d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params;
387c7e3548cSDan Carpenter unsigned int val, save;
388c7e3548cSDan Carpenter int rc;
38980d10a6cSBen Widawsky u8 iw;
39080d10a6cSBen Widawsky
391c7e3548cSDan Carpenter rc = kstrtouint(buf, 0, &val);
39280d10a6cSBen Widawsky if (rc)
39380d10a6cSBen Widawsky return rc;
39480d10a6cSBen Widawsky
395c99b2e8cSDave Jiang rc = ways_to_eiw(val, &iw);
39680d10a6cSBen Widawsky if (rc)
39780d10a6cSBen Widawsky return rc;
39880d10a6cSBen Widawsky
39980d10a6cSBen Widawsky /*
400a2b2b301SJim Harris * Even for x3, x6, and x12 interleaves the region interleave must be a
40180d10a6cSBen Widawsky * power of 2 multiple of the host bridge interleave.
40280d10a6cSBen Widawsky */
40380d10a6cSBen Widawsky if (!is_power_of_2(val / cxld->interleave_ways) ||
40480d10a6cSBen Widawsky (val % cxld->interleave_ways)) {
40580d10a6cSBen Widawsky dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
40680d10a6cSBen Widawsky return -EINVAL;
40780d10a6cSBen Widawsky }
40880d10a6cSBen Widawsky
40980d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem);
41080d10a6cSBen Widawsky if (rc)
41180d10a6cSBen Widawsky return rc;
41280d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
41380d10a6cSBen Widawsky rc = -EBUSY;
41480d10a6cSBen Widawsky goto out;
41580d10a6cSBen Widawsky }
41680d10a6cSBen Widawsky
417b9686e8cSDan Williams save = p->interleave_ways;
41880d10a6cSBen Widawsky p->interleave_ways = val;
419b9686e8cSDan Williams rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
420b9686e8cSDan Williams if (rc)
421b9686e8cSDan Williams p->interleave_ways = save;
42280d10a6cSBen Widawsky out:
42380d10a6cSBen Widawsky up_write(&cxl_region_rwsem);
42480d10a6cSBen Widawsky if (rc)
42580d10a6cSBen Widawsky return rc;
42680d10a6cSBen Widawsky return len;
42780d10a6cSBen Widawsky }
42880d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_ways);
42980d10a6cSBen Widawsky
interleave_granularity_show(struct device * dev,struct device_attribute * attr,char * buf)43080d10a6cSBen Widawsky static ssize_t interleave_granularity_show(struct device *dev,
43180d10a6cSBen Widawsky struct device_attribute *attr,
43280d10a6cSBen Widawsky char *buf)
43380d10a6cSBen Widawsky {
43480d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev);
43580d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params;
43680d10a6cSBen Widawsky ssize_t rc;
43780d10a6cSBen Widawsky
43880d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem);
43980d10a6cSBen Widawsky if (rc)
44080d10a6cSBen Widawsky return rc;
44180d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
44280d10a6cSBen Widawsky up_read(&cxl_region_rwsem);
44380d10a6cSBen Widawsky
44480d10a6cSBen Widawsky return rc;
44580d10a6cSBen Widawsky }
44680d10a6cSBen Widawsky
interleave_granularity_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)44780d10a6cSBen Widawsky static ssize_t interleave_granularity_store(struct device *dev,
44880d10a6cSBen Widawsky struct device_attribute *attr,
44980d10a6cSBen Widawsky const char *buf, size_t len)
45080d10a6cSBen Widawsky {
45180d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
45280d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
45380d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev);
45480d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params;
45580d10a6cSBen Widawsky int rc, val;
45680d10a6cSBen Widawsky u16 ig;
45780d10a6cSBen Widawsky
45880d10a6cSBen Widawsky rc = kstrtoint(buf, 0, &val);
45980d10a6cSBen Widawsky if (rc)
46080d10a6cSBen Widawsky return rc;
46180d10a6cSBen Widawsky
46283351ddbSDave Jiang rc = granularity_to_eig(val, &ig);
46380d10a6cSBen Widawsky if (rc)
46480d10a6cSBen Widawsky return rc;
46580d10a6cSBen Widawsky
46680d10a6cSBen Widawsky /*
4674d8e4ea5SDan Williams * When the host-bridge is interleaved, disallow region granularity !=
4684d8e4ea5SDan Williams * root granularity. Regions with a granularity less than the root
4694d8e4ea5SDan Williams * interleave result in needing multiple endpoints to support a single
470cbbd05d0SRandy Dunlap * slot in the interleave (possible to support in the future). Regions
4714d8e4ea5SDan Williams * with a granularity greater than the root interleave result in invalid
4724d8e4ea5SDan Williams * DPA translations (invalid to support).
47380d10a6cSBen Widawsky */
4744d8e4ea5SDan Williams if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
47580d10a6cSBen Widawsky return -EINVAL;
47680d10a6cSBen Widawsky
47780d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem);
47880d10a6cSBen Widawsky if (rc)
47980d10a6cSBen Widawsky return rc;
48080d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
48180d10a6cSBen Widawsky rc = -EBUSY;
48280d10a6cSBen Widawsky goto out;
48380d10a6cSBen Widawsky }
48480d10a6cSBen Widawsky
48580d10a6cSBen Widawsky p->interleave_granularity = val;
48680d10a6cSBen Widawsky out:
48780d10a6cSBen Widawsky up_write(&cxl_region_rwsem);
48880d10a6cSBen Widawsky if (rc)
48980d10a6cSBen Widawsky return rc;
49080d10a6cSBen Widawsky return len;
49180d10a6cSBen Widawsky }
49280d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_granularity);
49380d10a6cSBen Widawsky
resource_show(struct device * dev,struct device_attribute * attr,char * buf)49423a22cd1SDan Williams static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
49523a22cd1SDan Williams char *buf)
49623a22cd1SDan Williams {
49723a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev);
49823a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params;
49923a22cd1SDan Williams u64 resource = -1ULL;
50023a22cd1SDan Williams ssize_t rc;
50123a22cd1SDan Williams
50223a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem);
50323a22cd1SDan Williams if (rc)
50423a22cd1SDan Williams return rc;
50523a22cd1SDan Williams if (p->res)
50623a22cd1SDan Williams resource = p->res->start;
50723a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", resource);
50823a22cd1SDan Williams up_read(&cxl_region_rwsem);
50923a22cd1SDan Williams
51023a22cd1SDan Williams return rc;
51123a22cd1SDan Williams }
51223a22cd1SDan Williams static DEVICE_ATTR_RO(resource);
51323a22cd1SDan Williams
mode_show(struct device * dev,struct device_attribute * attr,char * buf)5147d505f98SDan Williams static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
5157d505f98SDan Williams char *buf)
5167d505f98SDan Williams {
5177d505f98SDan Williams struct cxl_region *cxlr = to_cxl_region(dev);
5187d505f98SDan Williams
5197d505f98SDan Williams return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
5207d505f98SDan Williams }
5217d505f98SDan Williams static DEVICE_ATTR_RO(mode);
5227d505f98SDan Williams
alloc_hpa(struct cxl_region * cxlr,resource_size_t size)52323a22cd1SDan Williams static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
52423a22cd1SDan Williams {
52523a22cd1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
52623a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params;
52723a22cd1SDan Williams struct resource *res;
52840cb184eSQuanquan Cao u64 remainder = 0;
52923a22cd1SDan Williams
53023a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem);
53123a22cd1SDan Williams
53223a22cd1SDan Williams /* Nothing to do... */
53388ab1ddeSDan Carpenter if (p->res && resource_size(p->res) == size)
53423a22cd1SDan Williams return 0;
53523a22cd1SDan Williams
53623a22cd1SDan Williams /* To change size the old size must be freed first */
53723a22cd1SDan Williams if (p->res)
53823a22cd1SDan Williams return -EBUSY;
53923a22cd1SDan Williams
54023a22cd1SDan Williams if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
54123a22cd1SDan Williams return -EBUSY;
54223a22cd1SDan Williams
54323a22cd1SDan Williams /* ways, granularity and uuid (if PMEM) need to be set before HPA */
54423a22cd1SDan Williams if (!p->interleave_ways || !p->interleave_granularity ||
54523a22cd1SDan Williams (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
54623a22cd1SDan Williams return -ENXIO;
54723a22cd1SDan Williams
54840cb184eSQuanquan Cao div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
54923a22cd1SDan Williams if (remainder)
55023a22cd1SDan Williams return -EINVAL;
55123a22cd1SDan Williams
55223a22cd1SDan Williams res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
55323a22cd1SDan Williams dev_name(&cxlr->dev));
55423a22cd1SDan Williams if (IS_ERR(res)) {
55523a22cd1SDan Williams dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
55623a22cd1SDan Williams PTR_ERR(res));
55723a22cd1SDan Williams return PTR_ERR(res);
55823a22cd1SDan Williams }
55923a22cd1SDan Williams
56023a22cd1SDan Williams p->res = res;
56123a22cd1SDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
56223a22cd1SDan Williams
56323a22cd1SDan Williams return 0;
56423a22cd1SDan Williams }
56523a22cd1SDan Williams
cxl_region_iomem_release(struct cxl_region * cxlr)56623a22cd1SDan Williams static void cxl_region_iomem_release(struct cxl_region *cxlr)
56723a22cd1SDan Williams {
56823a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params;
56923a22cd1SDan Williams
57023a22cd1SDan Williams if (device_is_registered(&cxlr->dev))
57123a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem);
57223a22cd1SDan Williams if (p->res) {
573a32320b7SDan Williams /*
574a32320b7SDan Williams * Autodiscovered regions may not have been able to insert their
575a32320b7SDan Williams * resource.
576a32320b7SDan Williams */
577a32320b7SDan Williams if (p->res->parent)
57823a22cd1SDan Williams remove_resource(p->res);
57923a22cd1SDan Williams kfree(p->res);
58023a22cd1SDan Williams p->res = NULL;
58123a22cd1SDan Williams }
58223a22cd1SDan Williams }
58323a22cd1SDan Williams
free_hpa(struct cxl_region * cxlr)58423a22cd1SDan Williams static int free_hpa(struct cxl_region *cxlr)
58523a22cd1SDan Williams {
58623a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params;
58723a22cd1SDan Williams
58823a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem);
58923a22cd1SDan Williams
59023a22cd1SDan Williams if (!p->res)
59123a22cd1SDan Williams return 0;
59223a22cd1SDan Williams
59323a22cd1SDan Williams if (p->state >= CXL_CONFIG_ACTIVE)
59423a22cd1SDan Williams return -EBUSY;
59523a22cd1SDan Williams
59623a22cd1SDan Williams cxl_region_iomem_release(cxlr);
59723a22cd1SDan Williams p->state = CXL_CONFIG_IDLE;
59823a22cd1SDan Williams return 0;
59923a22cd1SDan Williams }
60023a22cd1SDan Williams
size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)60123a22cd1SDan Williams static ssize_t size_store(struct device *dev, struct device_attribute *attr,
60223a22cd1SDan Williams const char *buf, size_t len)
60323a22cd1SDan Williams {
60423a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev);
60523a22cd1SDan Williams u64 val;
60623a22cd1SDan Williams int rc;
60723a22cd1SDan Williams
60823a22cd1SDan Williams rc = kstrtou64(buf, 0, &val);
60923a22cd1SDan Williams if (rc)
61023a22cd1SDan Williams return rc;
61123a22cd1SDan Williams
61223a22cd1SDan Williams rc = down_write_killable(&cxl_region_rwsem);
61323a22cd1SDan Williams if (rc)
61423a22cd1SDan Williams return rc;
61523a22cd1SDan Williams
61623a22cd1SDan Williams if (val)
61723a22cd1SDan Williams rc = alloc_hpa(cxlr, val);
61823a22cd1SDan Williams else
61923a22cd1SDan Williams rc = free_hpa(cxlr);
62023a22cd1SDan Williams up_write(&cxl_region_rwsem);
62123a22cd1SDan Williams
62223a22cd1SDan Williams if (rc)
62323a22cd1SDan Williams return rc;
62423a22cd1SDan Williams
62523a22cd1SDan Williams return len;
62623a22cd1SDan Williams }
62723a22cd1SDan Williams
size_show(struct device * dev,struct device_attribute * attr,char * buf)62823a22cd1SDan Williams static ssize_t size_show(struct device *dev, struct device_attribute *attr,
62923a22cd1SDan Williams char *buf)
63023a22cd1SDan Williams {
63123a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev);
63223a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params;
63323a22cd1SDan Williams u64 size = 0;
63423a22cd1SDan Williams ssize_t rc;
63523a22cd1SDan Williams
63623a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem);
63723a22cd1SDan Williams if (rc)
63823a22cd1SDan Williams return rc;
63923a22cd1SDan Williams if (p->res)
64023a22cd1SDan Williams size = resource_size(p->res);
64123a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", size);
64223a22cd1SDan Williams up_read(&cxl_region_rwsem);
64323a22cd1SDan Williams
64423a22cd1SDan Williams return rc;
64523a22cd1SDan Williams }
64623a22cd1SDan Williams static DEVICE_ATTR_RW(size);
64723a22cd1SDan Williams
648dd5ba0ebSBen Widawsky static struct attribute *cxl_region_attrs[] = {
649dd5ba0ebSBen Widawsky &dev_attr_uuid.attr,
650176baefbSDan Williams &dev_attr_commit.attr,
65180d10a6cSBen Widawsky &dev_attr_interleave_ways.attr,
65280d10a6cSBen Widawsky &dev_attr_interleave_granularity.attr,
65323a22cd1SDan Williams &dev_attr_resource.attr,
65423a22cd1SDan Williams &dev_attr_size.attr,
6557d505f98SDan Williams &dev_attr_mode.attr,
656dd5ba0ebSBen Widawsky NULL,
657dd5ba0ebSBen Widawsky };
658dd5ba0ebSBen Widawsky
659dd5ba0ebSBen Widawsky static const struct attribute_group cxl_region_group = {
660dd5ba0ebSBen Widawsky .attrs = cxl_region_attrs,
661dd5ba0ebSBen Widawsky .is_visible = cxl_region_visible,
662dd5ba0ebSBen Widawsky };
663dd5ba0ebSBen Widawsky
show_targetN(struct cxl_region * cxlr,char * buf,int pos)664b9686e8cSDan Williams static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
665b9686e8cSDan Williams {
666b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params;
667b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled;
668b9686e8cSDan Williams int rc;
669b9686e8cSDan Williams
670b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem);
671b9686e8cSDan Williams if (rc)
672b9686e8cSDan Williams return rc;
673b9686e8cSDan Williams
674b9686e8cSDan Williams if (pos >= p->interleave_ways) {
675b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
676b9686e8cSDan Williams p->interleave_ways);
677b9686e8cSDan Williams rc = -ENXIO;
678b9686e8cSDan Williams goto out;
679b9686e8cSDan Williams }
680b9686e8cSDan Williams
681b9686e8cSDan Williams cxled = p->targets[pos];
682b9686e8cSDan Williams if (!cxled)
683b9686e8cSDan Williams rc = sysfs_emit(buf, "\n");
684b9686e8cSDan Williams else
685b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
686b9686e8cSDan Williams out:
687b9686e8cSDan Williams up_read(&cxl_region_rwsem);
688b9686e8cSDan Williams
689b9686e8cSDan Williams return rc;
690b9686e8cSDan Williams }
691b9686e8cSDan Williams
match_free_decoder(struct device * dev,void * data)692384e624bSDan Williams static int match_free_decoder(struct device *dev, void *data)
693384e624bSDan Williams {
694384e624bSDan Williams struct cxl_decoder *cxld;
695384e624bSDan Williams int *id = data;
696384e624bSDan Williams
697384e624bSDan Williams if (!is_switch_decoder(dev))
698384e624bSDan Williams return 0;
699384e624bSDan Williams
700384e624bSDan Williams cxld = to_cxl_decoder(dev);
701384e624bSDan Williams
702384e624bSDan Williams /* enforce ordered allocation */
703384e624bSDan Williams if (cxld->id != *id)
704384e624bSDan Williams return 0;
705384e624bSDan Williams
706384e624bSDan Williams if (!cxld->region)
707384e624bSDan Williams return 1;
708384e624bSDan Williams
709384e624bSDan Williams (*id)++;
710384e624bSDan Williams
711384e624bSDan Williams return 0;
712384e624bSDan Williams }
713384e624bSDan Williams
match_auto_decoder(struct device * dev,void * data)7149e4edf1aSAlison Schofield static int match_auto_decoder(struct device *dev, void *data)
7159e4edf1aSAlison Schofield {
7169e4edf1aSAlison Schofield struct cxl_region_params *p = data;
7179e4edf1aSAlison Schofield struct cxl_decoder *cxld;
7189e4edf1aSAlison Schofield struct range *r;
7199e4edf1aSAlison Schofield
7209e4edf1aSAlison Schofield if (!is_switch_decoder(dev))
7219e4edf1aSAlison Schofield return 0;
7229e4edf1aSAlison Schofield
7239e4edf1aSAlison Schofield cxld = to_cxl_decoder(dev);
7249e4edf1aSAlison Schofield r = &cxld->hpa_range;
7259e4edf1aSAlison Schofield
7269e4edf1aSAlison Schofield if (p->res && p->res->start == r->start && p->res->end == r->end)
7279e4edf1aSAlison Schofield return 1;
7289e4edf1aSAlison Schofield
7299e4edf1aSAlison Schofield return 0;
7309e4edf1aSAlison Schofield }
7319e4edf1aSAlison Schofield
7329f57eecfSAlison Schofield static struct cxl_decoder *
cxl_region_find_decoder(struct cxl_port * port,struct cxl_endpoint_decoder * cxled,struct cxl_region * cxlr)7339f57eecfSAlison Schofield cxl_region_find_decoder(struct cxl_port *port,
7349f57eecfSAlison Schofield struct cxl_endpoint_decoder *cxled,
735384e624bSDan Williams struct cxl_region *cxlr)
736384e624bSDan Williams {
737384e624bSDan Williams struct device *dev;
738384e624bSDan Williams int id = 0;
739384e624bSDan Williams
7409f57eecfSAlison Schofield if (port == cxled_to_port(cxled))
7419f57eecfSAlison Schofield return &cxled->cxld;
7429f57eecfSAlison Schofield
7439e4edf1aSAlison Schofield if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
7449e4edf1aSAlison Schofield dev = device_find_child(&port->dev, &cxlr->params,
7459e4edf1aSAlison Schofield match_auto_decoder);
7469e4edf1aSAlison Schofield else
747384e624bSDan Williams dev = device_find_child(&port->dev, &id, match_free_decoder);
748384e624bSDan Williams if (!dev)
749384e624bSDan Williams return NULL;
750b9686e8cSDan Williams /*
751384e624bSDan Williams * This decoder is pinned registered as long as the endpoint decoder is
752384e624bSDan Williams * registered, and endpoint decoder unregistration holds the
753384e624bSDan Williams * cxl_region_rwsem over unregister events, so no need to hold on to
754384e624bSDan Williams * this extra reference.
755b9686e8cSDan Williams */
756384e624bSDan Williams put_device(dev);
757384e624bSDan Williams return to_cxl_decoder(dev);
758384e624bSDan Williams }
759384e624bSDan Williams
auto_order_ok(struct cxl_port * port,struct cxl_region * cxlr_iter,struct cxl_decoder * cxld)7600468ac56SAlison Schofield static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
7610468ac56SAlison Schofield struct cxl_decoder *cxld)
7620468ac56SAlison Schofield {
7630468ac56SAlison Schofield struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
7640468ac56SAlison Schofield struct cxl_decoder *cxld_iter = rr->decoder;
7650468ac56SAlison Schofield
7660468ac56SAlison Schofield /*
7670468ac56SAlison Schofield * Allow the out of order assembly of auto-discovered regions.
7680468ac56SAlison Schofield * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
7690468ac56SAlison Schofield * in HPA order. Confirm that the decoder with the lesser HPA
7700468ac56SAlison Schofield * starting address has the lesser id.
7710468ac56SAlison Schofield */
7720468ac56SAlison Schofield dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
7730468ac56SAlison Schofield dev_name(&cxld->dev), cxld->id,
7740468ac56SAlison Schofield dev_name(&cxld_iter->dev), cxld_iter->id);
7750468ac56SAlison Schofield
7760468ac56SAlison Schofield if (cxld_iter->id > cxld->id)
7770468ac56SAlison Schofield return true;
7780468ac56SAlison Schofield
7790468ac56SAlison Schofield return false;
7800468ac56SAlison Schofield }
7810468ac56SAlison Schofield
7820468ac56SAlison Schofield static struct cxl_region_ref *
alloc_region_ref(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)7830468ac56SAlison Schofield alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
7840468ac56SAlison Schofield struct cxl_endpoint_decoder *cxled)
785384e624bSDan Williams {
786e29a8995SDan Williams struct cxl_region_params *p = &cxlr->params;
787e29a8995SDan Williams struct cxl_region_ref *cxl_rr, *iter;
788e29a8995SDan Williams unsigned long index;
789384e624bSDan Williams int rc;
790384e624bSDan Williams
791e29a8995SDan Williams xa_for_each(&port->regions, index, iter) {
792e29a8995SDan Williams struct cxl_region_params *ip = &iter->region->params;
793e29a8995SDan Williams
7940468ac56SAlison Schofield if (!ip->res || ip->res->start < p->res->start)
795a90accb3SDan Williams continue;
796a90accb3SDan Williams
7970468ac56SAlison Schofield if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
7980468ac56SAlison Schofield struct cxl_decoder *cxld;
7990468ac56SAlison Schofield
8000468ac56SAlison Schofield cxld = cxl_region_find_decoder(port, cxled, cxlr);
8010468ac56SAlison Schofield if (auto_order_ok(port, iter->region, cxld))
8020468ac56SAlison Schofield continue;
8030468ac56SAlison Schofield }
8040468ac56SAlison Schofield dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
805e29a8995SDan Williams dev_name(&port->dev),
806e29a8995SDan Williams dev_name(&iter->region->dev), ip->res, p->res);
8070468ac56SAlison Schofield
808e29a8995SDan Williams return ERR_PTR(-EBUSY);
809e29a8995SDan Williams }
810e29a8995SDan Williams
811384e624bSDan Williams cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
812384e624bSDan Williams if (!cxl_rr)
813e29a8995SDan Williams return ERR_PTR(-ENOMEM);
814384e624bSDan Williams cxl_rr->port = port;
815384e624bSDan Williams cxl_rr->region = cxlr;
81627b3f8d1SDan Williams cxl_rr->nr_targets = 1;
817384e624bSDan Williams xa_init(&cxl_rr->endpoints);
818384e624bSDan Williams
819384e624bSDan Williams rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
820384e624bSDan Williams if (rc) {
821384e624bSDan Williams dev_dbg(&cxlr->dev,
822384e624bSDan Williams "%s: failed to track region reference: %d\n",
823384e624bSDan Williams dev_name(&port->dev), rc);
824384e624bSDan Williams kfree(cxl_rr);
825e29a8995SDan Williams return ERR_PTR(rc);
826384e624bSDan Williams }
827384e624bSDan Williams
828384e624bSDan Williams return cxl_rr;
829384e624bSDan Williams }
830384e624bSDan Williams
cxl_rr_free_decoder(struct cxl_region_ref * cxl_rr)83171ee71d7SVishal Verma static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
832384e624bSDan Williams {
833384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region;
834384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder;
835384e624bSDan Williams
83671ee71d7SVishal Verma if (!cxld)
83771ee71d7SVishal Verma return;
83871ee71d7SVishal Verma
839384e624bSDan Williams dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
840384e624bSDan Williams if (cxld->region == cxlr) {
841384e624bSDan Williams cxld->region = NULL;
842384e624bSDan Williams put_device(&cxlr->dev);
843384e624bSDan Williams }
84471ee71d7SVishal Verma }
845384e624bSDan Williams
free_region_ref(struct cxl_region_ref * cxl_rr)84671ee71d7SVishal Verma static void free_region_ref(struct cxl_region_ref *cxl_rr)
84771ee71d7SVishal Verma {
84871ee71d7SVishal Verma struct cxl_port *port = cxl_rr->port;
84971ee71d7SVishal Verma struct cxl_region *cxlr = cxl_rr->region;
85071ee71d7SVishal Verma
85171ee71d7SVishal Verma cxl_rr_free_decoder(cxl_rr);
852384e624bSDan Williams xa_erase(&port->regions, (unsigned long)cxlr);
853384e624bSDan Williams xa_destroy(&cxl_rr->endpoints);
854384e624bSDan Williams kfree(cxl_rr);
855384e624bSDan Williams }
856384e624bSDan Williams
cxl_rr_ep_add(struct cxl_region_ref * cxl_rr,struct cxl_endpoint_decoder * cxled)857384e624bSDan Williams static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
858384e624bSDan Williams struct cxl_endpoint_decoder *cxled)
859384e624bSDan Williams {
860384e624bSDan Williams int rc;
861384e624bSDan Williams struct cxl_port *port = cxl_rr->port;
862384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region;
863384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder;
864384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
865384e624bSDan Williams
86627b3f8d1SDan Williams if (ep) {
867384e624bSDan Williams rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
868384e624bSDan Williams GFP_KERNEL);
869384e624bSDan Williams if (rc)
870384e624bSDan Williams return rc;
87127b3f8d1SDan Williams }
872384e624bSDan Williams cxl_rr->nr_eps++;
873384e624bSDan Williams
874384e624bSDan Williams if (!cxld->region) {
875384e624bSDan Williams cxld->region = cxlr;
876384e624bSDan Williams get_device(&cxlr->dev);
877384e624bSDan Williams }
878384e624bSDan Williams
879384e624bSDan Williams return 0;
880384e624bSDan Williams }
881384e624bSDan Williams
cxl_rr_alloc_decoder(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,struct cxl_region_ref * cxl_rr)88271ee71d7SVishal Verma static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
88371ee71d7SVishal Verma struct cxl_endpoint_decoder *cxled,
88471ee71d7SVishal Verma struct cxl_region_ref *cxl_rr)
88571ee71d7SVishal Verma {
88671ee71d7SVishal Verma struct cxl_decoder *cxld;
88771ee71d7SVishal Verma
8889f57eecfSAlison Schofield cxld = cxl_region_find_decoder(port, cxled, cxlr);
88971ee71d7SVishal Verma if (!cxld) {
89071ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: no decoder available\n",
89171ee71d7SVishal Verma dev_name(&port->dev));
89271ee71d7SVishal Verma return -EBUSY;
89371ee71d7SVishal Verma }
89471ee71d7SVishal Verma
89571ee71d7SVishal Verma if (cxld->region) {
89671ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
89771ee71d7SVishal Verma dev_name(&port->dev), dev_name(&cxld->dev),
89871ee71d7SVishal Verma dev_name(&cxld->region->dev));
89971ee71d7SVishal Verma return -EBUSY;
90071ee71d7SVishal Verma }
90171ee71d7SVishal Verma
9028c897b36SDan Williams /*
9038c897b36SDan Williams * Endpoints should already match the region type, but backstop that
9048c897b36SDan Williams * assumption with an assertion. Switch-decoders change mapping-type
9058c897b36SDan Williams * based on what is mapped when they are assigned to a region.
9068c897b36SDan Williams */
9078c897b36SDan Williams dev_WARN_ONCE(&cxlr->dev,
9088c897b36SDan Williams port == cxled_to_port(cxled) &&
9098c897b36SDan Williams cxld->target_type != cxlr->type,
9108c897b36SDan Williams "%s:%s mismatch decoder type %d -> %d\n",
9118c897b36SDan Williams dev_name(&cxled_to_memdev(cxled)->dev),
9128c897b36SDan Williams dev_name(&cxld->dev), cxld->target_type, cxlr->type);
9138c897b36SDan Williams cxld->target_type = cxlr->type;
91471ee71d7SVishal Verma cxl_rr->decoder = cxld;
91571ee71d7SVishal Verma return 0;
91671ee71d7SVishal Verma }
91771ee71d7SVishal Verma
918384e624bSDan Williams /**
919384e624bSDan Williams * cxl_port_attach_region() - track a region's interest in a port by endpoint
920384e624bSDan Williams * @port: port to add a new region reference 'struct cxl_region_ref'
921384e624bSDan Williams * @cxlr: region to attach to @port
922384e624bSDan Williams * @cxled: endpoint decoder used to create or further pin a region reference
923384e624bSDan Williams * @pos: interleave position of @cxled in @cxlr
924384e624bSDan Williams *
925384e624bSDan Williams * The attach event is an opportunity to validate CXL decode setup
926384e624bSDan Williams * constraints and record metadata needed for programming HDM decoders,
927384e624bSDan Williams * in particular decoder target lists.
928384e624bSDan Williams *
929384e624bSDan Williams * The steps are:
930f13da0d9SBagas Sanjaya *
931384e624bSDan Williams * - validate that there are no other regions with a higher HPA already
932384e624bSDan Williams * associated with @port
933384e624bSDan Williams * - establish a region reference if one is not already present
934f13da0d9SBagas Sanjaya *
935384e624bSDan Williams * - additionally allocate a decoder instance that will host @cxlr on
936384e624bSDan Williams * @port
937f13da0d9SBagas Sanjaya *
938384e624bSDan Williams * - pin the region reference by the endpoint
939384e624bSDan Williams * - account for how many entries in @port's target list are needed to
940384e624bSDan Williams * cover all of the added endpoints.
941384e624bSDan Williams */
cxl_port_attach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)942384e624bSDan Williams static int cxl_port_attach_region(struct cxl_port *port,
943384e624bSDan Williams struct cxl_region *cxlr,
944384e624bSDan Williams struct cxl_endpoint_decoder *cxled, int pos)
945384e624bSDan Williams {
946384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
947384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
948e29a8995SDan Williams struct cxl_region_ref *cxl_rr;
949e29a8995SDan Williams bool nr_targets_inc = false;
950e29a8995SDan Williams struct cxl_decoder *cxld;
951384e624bSDan Williams unsigned long index;
952384e624bSDan Williams int rc = -EBUSY;
953384e624bSDan Williams
954384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem);
955384e624bSDan Williams
956e29a8995SDan Williams cxl_rr = cxl_rr_load(port, cxlr);
957384e624bSDan Williams if (cxl_rr) {
958384e624bSDan Williams struct cxl_ep *ep_iter;
959384e624bSDan Williams int found = 0;
960384e624bSDan Williams
961e29a8995SDan Williams /*
962e29a8995SDan Williams * Walk the existing endpoints that have been attached to
963e29a8995SDan Williams * @cxlr at @port and see if they share the same 'next' port
964e29a8995SDan Williams * in the downstream direction. I.e. endpoints that share common
965e29a8995SDan Williams * upstream switch.
966e29a8995SDan Williams */
967384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
968384e624bSDan Williams if (ep_iter == ep)
969384e624bSDan Williams continue;
970384e624bSDan Williams if (ep_iter->next == ep->next) {
971384e624bSDan Williams found++;
972384e624bSDan Williams break;
973384e624bSDan Williams }
974384e624bSDan Williams }
975384e624bSDan Williams
976384e624bSDan Williams /*
977e29a8995SDan Williams * New target port, or @port is an endpoint port that always
978e29a8995SDan Williams * accounts its own local decode as a target.
979384e624bSDan Williams */
980e29a8995SDan Williams if (!found || !ep->next) {
981384e624bSDan Williams cxl_rr->nr_targets++;
982e29a8995SDan Williams nr_targets_inc = true;
983e29a8995SDan Williams }
984384e624bSDan Williams } else {
9850468ac56SAlison Schofield cxl_rr = alloc_region_ref(port, cxlr, cxled);
986e29a8995SDan Williams if (IS_ERR(cxl_rr)) {
987384e624bSDan Williams dev_dbg(&cxlr->dev,
988384e624bSDan Williams "%s: failed to allocate region reference\n",
989384e624bSDan Williams dev_name(&port->dev));
990e29a8995SDan Williams return PTR_ERR(cxl_rr);
991384e624bSDan Williams }
992e29a8995SDan Williams nr_targets_inc = true;
993384e624bSDan Williams
99471ee71d7SVishal Verma rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
99571ee71d7SVishal Verma if (rc)
996384e624bSDan Williams goto out_erase;
997384e624bSDan Williams }
99871ee71d7SVishal Verma cxld = cxl_rr->decoder;
999384e624bSDan Williams
1000384e624bSDan Williams rc = cxl_rr_ep_add(cxl_rr, cxled);
1001384e624bSDan Williams if (rc) {
1002384e624bSDan Williams dev_dbg(&cxlr->dev,
1003384e624bSDan Williams "%s: failed to track endpoint %s:%s reference\n",
1004384e624bSDan Williams dev_name(&port->dev), dev_name(&cxlmd->dev),
1005384e624bSDan Williams dev_name(&cxld->dev));
1006384e624bSDan Williams goto out_erase;
1007384e624bSDan Williams }
1008384e624bSDan Williams
100927b3f8d1SDan Williams dev_dbg(&cxlr->dev,
101027b3f8d1SDan Williams "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
10117481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
101227b3f8d1SDan Williams dev_name(&cxld->dev), dev_name(&cxlmd->dev),
101327b3f8d1SDan Williams dev_name(&cxled->cxld.dev), pos,
10147481653dSDan Williams ep ? ep->next ? dev_name(ep->next->uport_dev) :
101527b3f8d1SDan Williams dev_name(&cxlmd->dev) :
101627b3f8d1SDan Williams "none",
101727b3f8d1SDan Williams cxl_rr->nr_eps, cxl_rr->nr_targets);
101827b3f8d1SDan Williams
1019384e624bSDan Williams return 0;
1020384e624bSDan Williams out_erase:
1021e29a8995SDan Williams if (nr_targets_inc)
1022e29a8995SDan Williams cxl_rr->nr_targets--;
1023384e624bSDan Williams if (cxl_rr->nr_eps == 0)
1024384e624bSDan Williams free_region_ref(cxl_rr);
1025384e624bSDan Williams return rc;
1026384e624bSDan Williams }
1027384e624bSDan Williams
cxl_port_detach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1028384e624bSDan Williams static void cxl_port_detach_region(struct cxl_port *port,
1029384e624bSDan Williams struct cxl_region *cxlr,
1030384e624bSDan Williams struct cxl_endpoint_decoder *cxled)
1031384e624bSDan Williams {
1032384e624bSDan Williams struct cxl_region_ref *cxl_rr;
103327b3f8d1SDan Williams struct cxl_ep *ep = NULL;
1034384e624bSDan Williams
1035384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem);
1036384e624bSDan Williams
1037384e624bSDan Williams cxl_rr = cxl_rr_load(port, cxlr);
1038384e624bSDan Williams if (!cxl_rr)
1039384e624bSDan Williams return;
1040384e624bSDan Williams
104127b3f8d1SDan Williams /*
104227b3f8d1SDan Williams * Endpoint ports do not carry cxl_ep references, and they
104327b3f8d1SDan Williams * never target more than one endpoint by definition
104427b3f8d1SDan Williams */
104527b3f8d1SDan Williams if (cxl_rr->decoder == &cxled->cxld)
104627b3f8d1SDan Williams cxl_rr->nr_eps--;
104727b3f8d1SDan Williams else
1048384e624bSDan Williams ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1049384e624bSDan Williams if (ep) {
1050384e624bSDan Williams struct cxl_ep *ep_iter;
1051384e624bSDan Williams unsigned long index;
1052384e624bSDan Williams int found = 0;
1053384e624bSDan Williams
1054384e624bSDan Williams cxl_rr->nr_eps--;
1055384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1056384e624bSDan Williams if (ep_iter->next == ep->next) {
1057384e624bSDan Williams found++;
1058384e624bSDan Williams break;
1059384e624bSDan Williams }
1060384e624bSDan Williams }
1061384e624bSDan Williams if (!found)
1062384e624bSDan Williams cxl_rr->nr_targets--;
1063384e624bSDan Williams }
1064384e624bSDan Williams
1065384e624bSDan Williams if (cxl_rr->nr_eps == 0)
1066384e624bSDan Williams free_region_ref(cxl_rr);
1067384e624bSDan Williams }
1068384e624bSDan Williams
check_last_peer(struct cxl_endpoint_decoder * cxled,struct cxl_ep * ep,struct cxl_region_ref * cxl_rr,int distance)106927b3f8d1SDan Williams static int check_last_peer(struct cxl_endpoint_decoder *cxled,
107027b3f8d1SDan Williams struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
107127b3f8d1SDan Williams int distance)
107227b3f8d1SDan Williams {
107327b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
107427b3f8d1SDan Williams struct cxl_region *cxlr = cxl_rr->region;
107527b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params;
107627b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled_peer;
107727b3f8d1SDan Williams struct cxl_port *port = cxl_rr->port;
107827b3f8d1SDan Williams struct cxl_memdev *cxlmd_peer;
107927b3f8d1SDan Williams struct cxl_ep *ep_peer;
108027b3f8d1SDan Williams int pos = cxled->pos;
108127b3f8d1SDan Williams
108227b3f8d1SDan Williams /*
108327b3f8d1SDan Williams * If this position wants to share a dport with the last endpoint mapped
108427b3f8d1SDan Williams * then that endpoint, at index 'position - distance', must also be
108527b3f8d1SDan Williams * mapped by this dport.
108627b3f8d1SDan Williams */
108727b3f8d1SDan Williams if (pos < distance) {
108827b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
10897481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
109027b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
109127b3f8d1SDan Williams return -ENXIO;
109227b3f8d1SDan Williams }
109327b3f8d1SDan Williams cxled_peer = p->targets[pos - distance];
109427b3f8d1SDan Williams cxlmd_peer = cxled_to_memdev(cxled_peer);
109527b3f8d1SDan Williams ep_peer = cxl_ep_load(port, cxlmd_peer);
109627b3f8d1SDan Williams if (ep->dport != ep_peer->dport) {
109727b3f8d1SDan Williams dev_dbg(&cxlr->dev,
109827b3f8d1SDan Williams "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
10997481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
110027b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
110127b3f8d1SDan Williams dev_name(&cxlmd_peer->dev),
110227b3f8d1SDan Williams dev_name(&cxled_peer->cxld.dev));
110327b3f8d1SDan Williams return -ENXIO;
110427b3f8d1SDan Williams }
110527b3f8d1SDan Williams
110627b3f8d1SDan Williams return 0;
110727b3f8d1SDan Williams }
110827b3f8d1SDan Williams
cxl_port_setup_targets(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)110927b3f8d1SDan Williams static int cxl_port_setup_targets(struct cxl_port *port,
111027b3f8d1SDan Williams struct cxl_region *cxlr,
111127b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled)
111227b3f8d1SDan Williams {
111327b3f8d1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
111427b3f8d1SDan Williams int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
111527b3f8d1SDan Williams struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
111627b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
111727b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
111827b3f8d1SDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
111927b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params;
112027b3f8d1SDan Williams struct cxl_decoder *cxld = cxl_rr->decoder;
112127b3f8d1SDan Williams struct cxl_switch_decoder *cxlsd;
112227b3f8d1SDan Williams u16 eig, peig;
112327b3f8d1SDan Williams u8 eiw, peiw;
112427b3f8d1SDan Williams
112527b3f8d1SDan Williams /*
112627b3f8d1SDan Williams * While root level decoders support x3, x6, x12, switch level
112727b3f8d1SDan Williams * decoders only support powers of 2 up to x16.
112827b3f8d1SDan Williams */
112927b3f8d1SDan Williams if (!is_power_of_2(cxl_rr->nr_targets)) {
113027b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
11317481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
113227b3f8d1SDan Williams cxl_rr->nr_targets);
113327b3f8d1SDan Williams return -EINVAL;
113427b3f8d1SDan Williams }
113527b3f8d1SDan Williams
113627b3f8d1SDan Williams cxlsd = to_cxl_switch_decoder(&cxld->dev);
113727b3f8d1SDan Williams if (cxl_rr->nr_targets_set) {
113827b3f8d1SDan Williams int i, distance;
113927b3f8d1SDan Williams
1140e4f6dfa9SDan Williams /*
1141711442e2SDan Williams * Passthrough decoders impose no distance requirements between
1142e4f6dfa9SDan Williams * peers
1143e4f6dfa9SDan Williams */
1144711442e2SDan Williams if (cxl_rr->nr_targets == 1)
1145e4f6dfa9SDan Williams distance = 0;
1146e4f6dfa9SDan Williams else
114727b3f8d1SDan Williams distance = p->nr_targets / cxl_rr->nr_targets;
114827b3f8d1SDan Williams for (i = 0; i < cxl_rr->nr_targets_set; i++)
114927b3f8d1SDan Williams if (ep->dport == cxlsd->target[i]) {
115027b3f8d1SDan Williams rc = check_last_peer(cxled, ep, cxl_rr,
115127b3f8d1SDan Williams distance);
115227b3f8d1SDan Williams if (rc)
115327b3f8d1SDan Williams return rc;
115427b3f8d1SDan Williams goto out_target_set;
115527b3f8d1SDan Williams }
115627b3f8d1SDan Williams goto add_target;
115727b3f8d1SDan Williams }
115827b3f8d1SDan Williams
115927b3f8d1SDan Williams if (is_cxl_root(parent_port)) {
11608a9ab903SJim Harris /*
11618a9ab903SJim Harris * Root decoder IG is always set to value in CFMWS which
11628a9ab903SJim Harris * may be different than this region's IG. We can use the
11638a9ab903SJim Harris * region's IG here since interleave_granularity_store()
11648a9ab903SJim Harris * does not allow interleaved host-bridges with
11658a9ab903SJim Harris * root IG != region IG.
11668a9ab903SJim Harris */
11678a9ab903SJim Harris parent_ig = p->interleave_granularity;
116827b3f8d1SDan Williams parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
116927b3f8d1SDan Williams /*
117027b3f8d1SDan Williams * For purposes of address bit routing, use power-of-2 math for
117127b3f8d1SDan Williams * switch ports.
117227b3f8d1SDan Williams */
117327b3f8d1SDan Williams if (!is_power_of_2(parent_iw))
117427b3f8d1SDan Williams parent_iw /= 3;
117527b3f8d1SDan Williams } else {
117627b3f8d1SDan Williams struct cxl_region_ref *parent_rr;
117727b3f8d1SDan Williams struct cxl_decoder *parent_cxld;
117827b3f8d1SDan Williams
117927b3f8d1SDan Williams parent_rr = cxl_rr_load(parent_port, cxlr);
118027b3f8d1SDan Williams parent_cxld = parent_rr->decoder;
118127b3f8d1SDan Williams parent_ig = parent_cxld->interleave_granularity;
118227b3f8d1SDan Williams parent_iw = parent_cxld->interleave_ways;
118327b3f8d1SDan Williams }
118427b3f8d1SDan Williams
118583351ddbSDave Jiang rc = granularity_to_eig(parent_ig, &peig);
11868d428542SDan Williams if (rc) {
11878d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
11887481653dSDan Williams dev_name(parent_port->uport_dev),
11898d428542SDan Williams dev_name(&parent_port->dev), parent_ig);
11908d428542SDan Williams return rc;
11918d428542SDan Williams }
11928d428542SDan Williams
1193c99b2e8cSDave Jiang rc = ways_to_eiw(parent_iw, &peiw);
11948d428542SDan Williams if (rc) {
11958d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
11967481653dSDan Williams dev_name(parent_port->uport_dev),
11978d428542SDan Williams dev_name(&parent_port->dev), parent_iw);
11988d428542SDan Williams return rc;
11998d428542SDan Williams }
120027b3f8d1SDan Williams
120127b3f8d1SDan Williams iw = cxl_rr->nr_targets;
1202c99b2e8cSDave Jiang rc = ways_to_eiw(iw, &eiw);
12038d428542SDan Williams if (rc) {
12048d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
12057481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), iw);
12068d428542SDan Williams return rc;
12078d428542SDan Williams }
12088d428542SDan Williams
1209298d44d0SDan Williams /*
121018f35dc9SAlison Schofield * Interleave granularity is a multiple of @parent_port granularity.
121118f35dc9SAlison Schofield * Multiplier is the parent port interleave ways.
1212298d44d0SDan Williams */
121318f35dc9SAlison Schofield rc = granularity_to_eig(parent_ig * parent_iw, &eig);
121418f35dc9SAlison Schofield if (rc) {
121518f35dc9SAlison Schofield dev_dbg(&cxlr->dev,
121618f35dc9SAlison Schofield "%s: invalid granularity calculation (%d * %d)\n",
121718f35dc9SAlison Schofield dev_name(&parent_port->dev), parent_ig, parent_iw);
121818f35dc9SAlison Schofield return rc;
121927b3f8d1SDan Williams }
122027b3f8d1SDan Williams
122183351ddbSDave Jiang rc = eig_to_granularity(eig, &ig);
122227b3f8d1SDan Williams if (rc) {
122327b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
12247481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
122527b3f8d1SDan Williams 256 << eig);
122627b3f8d1SDan Williams return rc;
122727b3f8d1SDan Williams }
122827b3f8d1SDan Williams
12296723e58dSDan Williams if (iw > 8 || iw > cxlsd->nr_targets) {
12306723e58dSDan Williams dev_dbg(&cxlr->dev,
12316723e58dSDan Williams "%s:%s:%s: ways: %d overflows targets: %d\n",
12326723e58dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
12336723e58dSDan Williams dev_name(&cxld->dev), iw, cxlsd->nr_targets);
12346723e58dSDan Williams return -ENXIO;
12356723e58dSDan Williams }
12366723e58dSDan Williams
1237a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1238a32320b7SDan Williams if (cxld->interleave_ways != iw ||
1239a32320b7SDan Williams cxld->interleave_granularity != ig ||
1240a32320b7SDan Williams cxld->hpa_range.start != p->res->start ||
1241a32320b7SDan Williams cxld->hpa_range.end != p->res->end ||
1242a32320b7SDan Williams ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1243a32320b7SDan Williams dev_err(&cxlr->dev,
1244a32320b7SDan Williams "%s:%s %s expected iw: %d ig: %d %pr\n",
12457481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
1246a32320b7SDan Williams __func__, iw, ig, p->res);
1247a32320b7SDan Williams dev_err(&cxlr->dev,
1248a32320b7SDan Williams "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
12497481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
1250a32320b7SDan Williams __func__, cxld->interleave_ways,
1251a32320b7SDan Williams cxld->interleave_granularity,
1252a32320b7SDan Williams (cxld->flags & CXL_DECODER_F_ENABLE) ?
1253a32320b7SDan Williams "enabled" :
1254a32320b7SDan Williams "disabled",
1255a32320b7SDan Williams cxld->hpa_range.start, cxld->hpa_range.end);
1256a32320b7SDan Williams return -ENXIO;
1257a32320b7SDan Williams }
1258a32320b7SDan Williams } else {
125927b3f8d1SDan Williams cxld->interleave_ways = iw;
126027b3f8d1SDan Williams cxld->interleave_granularity = ig;
1261910bc55dSDan Williams cxld->hpa_range = (struct range) {
1262910bc55dSDan Williams .start = p->res->start,
1263910bc55dSDan Williams .end = p->res->end,
1264910bc55dSDan Williams };
1265a32320b7SDan Williams }
12667481653dSDan Williams dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
126727b3f8d1SDan Williams dev_name(&port->dev), iw, ig);
126827b3f8d1SDan Williams add_target:
126927b3f8d1SDan Williams if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
127027b3f8d1SDan Williams dev_dbg(&cxlr->dev,
127127b3f8d1SDan Williams "%s:%s: targets full trying to add %s:%s at %d\n",
12727481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
127327b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
127427b3f8d1SDan Williams return -ENXIO;
127527b3f8d1SDan Williams }
1276a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1277a32320b7SDan Williams if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1278a32320b7SDan Williams dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
12797481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
1280a32320b7SDan Williams dev_name(&cxlsd->cxld.dev),
1281227db574SRobert Richter dev_name(ep->dport->dport_dev),
1282a32320b7SDan Williams cxl_rr->nr_targets_set);
1283a32320b7SDan Williams return -ENXIO;
1284a32320b7SDan Williams }
1285a32320b7SDan Williams } else
128627b3f8d1SDan Williams cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
128727b3f8d1SDan Williams inc = 1;
128827b3f8d1SDan Williams out_target_set:
128927b3f8d1SDan Williams cxl_rr->nr_targets_set += inc;
129027b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
12917481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev),
1292227db574SRobert Richter cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
129327b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
129427b3f8d1SDan Williams
129527b3f8d1SDan Williams return 0;
129627b3f8d1SDan Williams }
129727b3f8d1SDan Williams
cxl_port_reset_targets(struct cxl_port * port,struct cxl_region * cxlr)129827b3f8d1SDan Williams static void cxl_port_reset_targets(struct cxl_port *port,
129927b3f8d1SDan Williams struct cxl_region *cxlr)
130027b3f8d1SDan Williams {
130127b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1302910bc55dSDan Williams struct cxl_decoder *cxld;
130327b3f8d1SDan Williams
130427b3f8d1SDan Williams /*
130527b3f8d1SDan Williams * After the last endpoint has been detached the entire cxl_rr may now
130627b3f8d1SDan Williams * be gone.
130727b3f8d1SDan Williams */
1308910bc55dSDan Williams if (!cxl_rr)
1309910bc55dSDan Williams return;
131027b3f8d1SDan Williams cxl_rr->nr_targets_set = 0;
1311910bc55dSDan Williams
1312910bc55dSDan Williams cxld = cxl_rr->decoder;
1313910bc55dSDan Williams cxld->hpa_range = (struct range) {
1314910bc55dSDan Williams .start = 0,
1315910bc55dSDan Williams .end = -1,
1316910bc55dSDan Williams };
131727b3f8d1SDan Williams }
131827b3f8d1SDan Williams
cxl_region_teardown_targets(struct cxl_region * cxlr)131927b3f8d1SDan Williams static void cxl_region_teardown_targets(struct cxl_region *cxlr)
132027b3f8d1SDan Williams {
132127b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params;
132227b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled;
1323030f8803SDan Williams struct cxl_dev_state *cxlds;
132427b3f8d1SDan Williams struct cxl_memdev *cxlmd;
132527b3f8d1SDan Williams struct cxl_port *iter;
132627b3f8d1SDan Williams struct cxl_ep *ep;
132727b3f8d1SDan Williams int i;
132827b3f8d1SDan Williams
1329a32320b7SDan Williams /*
1330a32320b7SDan Williams * In the auto-discovery case skip automatic teardown since the
1331a32320b7SDan Williams * address space is already active
1332a32320b7SDan Williams */
1333a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1334a32320b7SDan Williams return;
1335a32320b7SDan Williams
133627b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) {
133727b3f8d1SDan Williams cxled = p->targets[i];
133827b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled);
1339030f8803SDan Williams cxlds = cxlmd->cxlds;
1340030f8803SDan Williams
1341030f8803SDan Williams if (cxlds->rcd)
1342030f8803SDan Williams continue;
134327b3f8d1SDan Williams
134427b3f8d1SDan Williams iter = cxled_to_port(cxled);
134527b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
134627b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent);
134727b3f8d1SDan Williams
134827b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter;
134927b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
135027b3f8d1SDan Williams cxl_port_reset_targets(iter, cxlr);
135127b3f8d1SDan Williams }
135227b3f8d1SDan Williams }
135327b3f8d1SDan Williams
cxl_region_setup_targets(struct cxl_region * cxlr)135427b3f8d1SDan Williams static int cxl_region_setup_targets(struct cxl_region *cxlr)
135527b3f8d1SDan Williams {
135627b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params;
135727b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled;
1358030f8803SDan Williams struct cxl_dev_state *cxlds;
1359030f8803SDan Williams int i, rc, rch = 0, vh = 0;
136027b3f8d1SDan Williams struct cxl_memdev *cxlmd;
136127b3f8d1SDan Williams struct cxl_port *iter;
136227b3f8d1SDan Williams struct cxl_ep *ep;
136327b3f8d1SDan Williams
136427b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) {
136527b3f8d1SDan Williams cxled = p->targets[i];
136627b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled);
1367030f8803SDan Williams cxlds = cxlmd->cxlds;
1368030f8803SDan Williams
1369030f8803SDan Williams /* validate that all targets agree on topology */
1370030f8803SDan Williams if (!cxlds->rcd) {
1371030f8803SDan Williams vh++;
1372030f8803SDan Williams } else {
1373030f8803SDan Williams rch++;
1374030f8803SDan Williams continue;
1375030f8803SDan Williams }
137627b3f8d1SDan Williams
137727b3f8d1SDan Williams iter = cxled_to_port(cxled);
137827b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
137927b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent);
138027b3f8d1SDan Williams
138127b3f8d1SDan Williams /*
1382a32320b7SDan Williams * Descend the topology tree programming / validating
1383a32320b7SDan Williams * targets while looking for conflicts.
138427b3f8d1SDan Williams */
138527b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter;
138627b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
138727b3f8d1SDan Williams rc = cxl_port_setup_targets(iter, cxlr, cxled);
138827b3f8d1SDan Williams if (rc) {
138927b3f8d1SDan Williams cxl_region_teardown_targets(cxlr);
139027b3f8d1SDan Williams return rc;
139127b3f8d1SDan Williams }
139227b3f8d1SDan Williams }
139327b3f8d1SDan Williams }
139427b3f8d1SDan Williams
1395030f8803SDan Williams if (rch && vh) {
1396030f8803SDan Williams dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1397030f8803SDan Williams cxl_region_teardown_targets(cxlr);
1398030f8803SDan Williams return -ENXIO;
1399030f8803SDan Williams }
1400030f8803SDan Williams
140127b3f8d1SDan Williams return 0;
140227b3f8d1SDan Williams }
140327b3f8d1SDan Williams
cxl_region_validate_position(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)14049995576cSDan Williams static int cxl_region_validate_position(struct cxl_region *cxlr,
14059995576cSDan Williams struct cxl_endpoint_decoder *cxled,
14069995576cSDan Williams int pos)
1407b9686e8cSDan Williams {
1408384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1409b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params;
14109995576cSDan Williams int i;
1411384e624bSDan Williams
1412384e624bSDan Williams if (pos < 0 || pos >= p->interleave_ways) {
1413b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1414b9686e8cSDan Williams p->interleave_ways);
1415b9686e8cSDan Williams return -ENXIO;
1416b9686e8cSDan Williams }
1417b9686e8cSDan Williams
1418b9686e8cSDan Williams if (p->targets[pos] == cxled)
1419b9686e8cSDan Williams return 0;
1420b9686e8cSDan Williams
1421b9686e8cSDan Williams if (p->targets[pos]) {
1422b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1423b9686e8cSDan Williams struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1424b9686e8cSDan Williams
1425b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1426b9686e8cSDan Williams pos, dev_name(&cxlmd_target->dev),
1427b9686e8cSDan Williams dev_name(&cxled_target->cxld.dev));
1428b9686e8cSDan Williams return -EBUSY;
1429b9686e8cSDan Williams }
1430b9686e8cSDan Williams
1431384e624bSDan Williams for (i = 0; i < p->interleave_ways; i++) {
1432384e624bSDan Williams struct cxl_endpoint_decoder *cxled_target;
1433384e624bSDan Williams struct cxl_memdev *cxlmd_target;
1434384e624bSDan Williams
1435f04facfbSFan Ni cxled_target = p->targets[i];
1436384e624bSDan Williams if (!cxled_target)
1437384e624bSDan Williams continue;
1438384e624bSDan Williams
1439384e624bSDan Williams cxlmd_target = cxled_to_memdev(cxled_target);
1440384e624bSDan Williams if (cxlmd_target == cxlmd) {
1441384e624bSDan Williams dev_dbg(&cxlr->dev,
1442384e624bSDan Williams "%s already specified at position %d via: %s\n",
1443384e624bSDan Williams dev_name(&cxlmd->dev), pos,
1444384e624bSDan Williams dev_name(&cxled_target->cxld.dev));
1445384e624bSDan Williams return -EBUSY;
1446384e624bSDan Williams }
1447384e624bSDan Williams }
1448384e624bSDan Williams
14499995576cSDan Williams return 0;
14509995576cSDan Williams }
14519995576cSDan Williams
cxl_region_attach_position(struct cxl_region * cxlr,struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled,const struct cxl_dport * dport,int pos)14529995576cSDan Williams static int cxl_region_attach_position(struct cxl_region *cxlr,
14539995576cSDan Williams struct cxl_root_decoder *cxlrd,
14549995576cSDan Williams struct cxl_endpoint_decoder *cxled,
14559995576cSDan Williams const struct cxl_dport *dport, int pos)
14569995576cSDan Williams {
14579995576cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
14589995576cSDan Williams struct cxl_port *iter;
14599995576cSDan Williams int rc;
14609995576cSDan Williams
14619995576cSDan Williams if (cxlrd->calc_hb(cxlrd, pos) != dport) {
14629995576cSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
14639995576cSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
14649995576cSDan Williams dev_name(&cxlrd->cxlsd.cxld.dev));
14659995576cSDan Williams return -ENXIO;
14669995576cSDan Williams }
14679995576cSDan Williams
14689995576cSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
14699995576cSDan Williams iter = to_cxl_port(iter->dev.parent)) {
14709995576cSDan Williams rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
14719995576cSDan Williams if (rc)
14729995576cSDan Williams goto err;
14739995576cSDan Williams }
14749995576cSDan Williams
14759995576cSDan Williams return 0;
14769995576cSDan Williams
14779995576cSDan Williams err:
14789995576cSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
14799995576cSDan Williams iter = to_cxl_port(iter->dev.parent))
14809995576cSDan Williams cxl_port_detach_region(iter, cxlr, cxled);
14819995576cSDan Williams return rc;
14829995576cSDan Williams }
14839995576cSDan Williams
cxl_region_attach_auto(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1484a32320b7SDan Williams static int cxl_region_attach_auto(struct cxl_region *cxlr,
1485a32320b7SDan Williams struct cxl_endpoint_decoder *cxled, int pos)
1486a32320b7SDan Williams {
1487a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params;
1488a32320b7SDan Williams
1489a32320b7SDan Williams if (cxled->state != CXL_DECODER_STATE_AUTO) {
1490a32320b7SDan Williams dev_err(&cxlr->dev,
1491a32320b7SDan Williams "%s: unable to add decoder to autodetected region\n",
1492a32320b7SDan Williams dev_name(&cxled->cxld.dev));
1493a32320b7SDan Williams return -EINVAL;
1494a32320b7SDan Williams }
1495a32320b7SDan Williams
1496a32320b7SDan Williams if (pos >= 0) {
1497a32320b7SDan Williams dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1498a32320b7SDan Williams dev_name(&cxled->cxld.dev), pos);
1499a32320b7SDan Williams return -EINVAL;
1500a32320b7SDan Williams }
1501a32320b7SDan Williams
1502a32320b7SDan Williams if (p->nr_targets >= p->interleave_ways) {
1503a32320b7SDan Williams dev_err(&cxlr->dev, "%s: no more target slots available\n",
1504a32320b7SDan Williams dev_name(&cxled->cxld.dev));
1505a32320b7SDan Williams return -ENXIO;
1506a32320b7SDan Williams }
1507a32320b7SDan Williams
1508a32320b7SDan Williams /*
1509a32320b7SDan Williams * Temporarily record the endpoint decoder into the target array. Yes,
1510a32320b7SDan Williams * this means that userspace can view devices in the wrong position
1511a32320b7SDan Williams * before the region activates, and must be careful to understand when
1512a32320b7SDan Williams * it might be racing region autodiscovery.
1513a32320b7SDan Williams */
1514a32320b7SDan Williams pos = p->nr_targets;
1515a32320b7SDan Williams p->targets[pos] = cxled;
1516a32320b7SDan Williams cxled->pos = pos;
1517a32320b7SDan Williams p->nr_targets++;
1518a32320b7SDan Williams
1519a32320b7SDan Williams return 0;
1520a32320b7SDan Williams }
1521a32320b7SDan Williams
cmp_interleave_pos(const void * a,const void * b)15223cfdfce0SAlison Schofield static int cmp_interleave_pos(const void *a, const void *b)
15233cfdfce0SAlison Schofield {
15243cfdfce0SAlison Schofield struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
15253cfdfce0SAlison Schofield struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
15263cfdfce0SAlison Schofield
15273cfdfce0SAlison Schofield return cxled_a->pos - cxled_b->pos;
15283cfdfce0SAlison Schofield }
15293cfdfce0SAlison Schofield
next_port(struct cxl_port * port)1530a32320b7SDan Williams static struct cxl_port *next_port(struct cxl_port *port)
1531a32320b7SDan Williams {
1532a32320b7SDan Williams if (!port->parent_dport)
1533a32320b7SDan Williams return NULL;
1534a32320b7SDan Williams return port->parent_dport->port;
1535a32320b7SDan Williams }
1536a32320b7SDan Williams
match_switch_decoder_by_range(struct device * dev,void * data)1537c4255b9bSAlison Schofield static int match_switch_decoder_by_range(struct device *dev, void *data)
1538a32320b7SDan Williams {
1539a32320b7SDan Williams struct cxl_switch_decoder *cxlsd;
1540c4255b9bSAlison Schofield struct range *r1, *r2 = data;
1541a32320b7SDan Williams
1542a32320b7SDan Williams if (!is_switch_decoder(dev))
1543a32320b7SDan Williams return 0;
1544a32320b7SDan Williams
1545a32320b7SDan Williams cxlsd = to_cxl_switch_decoder(dev);
1546c4255b9bSAlison Schofield r1 = &cxlsd->cxld.hpa_range;
1547c4255b9bSAlison Schofield
1548c4255b9bSAlison Schofield if (is_root_decoder(dev))
1549c4255b9bSAlison Schofield return range_contains(r1, r2);
1550c4255b9bSAlison Schofield return (r1->start == r2->start && r1->end == r2->end);
1551a32320b7SDan Williams }
1552a32320b7SDan Williams
find_pos_and_ways(struct cxl_port * port,struct range * range,int * pos,int * ways)1553c6ffabc6SAlison Schofield static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1554c6ffabc6SAlison Schofield int *pos, int *ways)
1555c6ffabc6SAlison Schofield {
1556c6ffabc6SAlison Schofield struct cxl_switch_decoder *cxlsd;
1557c6ffabc6SAlison Schofield struct cxl_port *parent;
1558c6ffabc6SAlison Schofield struct device *dev;
1559c6ffabc6SAlison Schofield int rc = -ENXIO;
1560c6ffabc6SAlison Schofield
1561c6ffabc6SAlison Schofield parent = next_port(port);
1562c6ffabc6SAlison Schofield if (!parent)
1563c6ffabc6SAlison Schofield return rc;
1564c6ffabc6SAlison Schofield
1565c6ffabc6SAlison Schofield dev = device_find_child(&parent->dev, range,
1566c6ffabc6SAlison Schofield match_switch_decoder_by_range);
1567c6ffabc6SAlison Schofield if (!dev) {
1568c6ffabc6SAlison Schofield dev_err(port->uport_dev,
1569c6ffabc6SAlison Schofield "failed to find decoder mapping %#llx-%#llx\n",
1570c6ffabc6SAlison Schofield range->start, range->end);
1571c6ffabc6SAlison Schofield return rc;
1572c6ffabc6SAlison Schofield }
1573c6ffabc6SAlison Schofield cxlsd = to_cxl_switch_decoder(dev);
1574c6ffabc6SAlison Schofield *ways = cxlsd->cxld.interleave_ways;
1575c6ffabc6SAlison Schofield
1576c6ffabc6SAlison Schofield for (int i = 0; i < *ways; i++) {
1577c6ffabc6SAlison Schofield if (cxlsd->target[i] == port->parent_dport) {
1578c6ffabc6SAlison Schofield *pos = i;
1579c6ffabc6SAlison Schofield rc = 0;
1580c6ffabc6SAlison Schofield break;
1581c6ffabc6SAlison Schofield }
1582c6ffabc6SAlison Schofield }
1583c6ffabc6SAlison Schofield put_device(dev);
1584c6ffabc6SAlison Schofield
1585c6ffabc6SAlison Schofield return rc;
1586c6ffabc6SAlison Schofield }
1587c6ffabc6SAlison Schofield
1588c6ffabc6SAlison Schofield /**
1589c6ffabc6SAlison Schofield * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1590c6ffabc6SAlison Schofield * @cxled: endpoint decoder member of given region
1591c6ffabc6SAlison Schofield *
1592c6ffabc6SAlison Schofield * The endpoint position is calculated by traversing the topology from
1593c6ffabc6SAlison Schofield * the endpoint to the root decoder and iteratively applying this
1594c6ffabc6SAlison Schofield * calculation:
1595c6ffabc6SAlison Schofield *
1596c6ffabc6SAlison Schofield * position = position * parent_ways + parent_pos;
1597c6ffabc6SAlison Schofield *
1598c6ffabc6SAlison Schofield * ...where @position is inferred from switch and root decoder target lists.
1599c6ffabc6SAlison Schofield *
1600c6ffabc6SAlison Schofield * Return: position >= 0 on success
1601c6ffabc6SAlison Schofield * -ENXIO on failure
1602c6ffabc6SAlison Schofield */
cxl_calc_interleave_pos(struct cxl_endpoint_decoder * cxled)1603c6ffabc6SAlison Schofield static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1604c6ffabc6SAlison Schofield {
1605c6ffabc6SAlison Schofield struct cxl_port *iter, *port = cxled_to_port(cxled);
1606c6ffabc6SAlison Schofield struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1607c6ffabc6SAlison Schofield struct range *range = &cxled->cxld.hpa_range;
1608c6ffabc6SAlison Schofield int parent_ways = 0, parent_pos = 0, pos = 0;
1609c6ffabc6SAlison Schofield int rc;
1610c6ffabc6SAlison Schofield
1611c6ffabc6SAlison Schofield /*
1612c6ffabc6SAlison Schofield * Example: the expected interleave order of the 4-way region shown
1613c6ffabc6SAlison Schofield * below is: mem0, mem2, mem1, mem3
1614c6ffabc6SAlison Schofield *
1615c6ffabc6SAlison Schofield * root_port
1616c6ffabc6SAlison Schofield * / \
1617c6ffabc6SAlison Schofield * host_bridge_0 host_bridge_1
1618c6ffabc6SAlison Schofield * | | | |
1619c6ffabc6SAlison Schofield * mem0 mem1 mem2 mem3
1620c6ffabc6SAlison Schofield *
1621c6ffabc6SAlison Schofield * In the example the calculator will iterate twice. The first iteration
1622c6ffabc6SAlison Schofield * uses the mem position in the host-bridge and the ways of the host-
1623c6ffabc6SAlison Schofield * bridge to generate the first, or local, position. The second
1624c6ffabc6SAlison Schofield * iteration uses the host-bridge position in the root_port and the ways
1625c6ffabc6SAlison Schofield * of the root_port to refine the position.
1626c6ffabc6SAlison Schofield *
1627c6ffabc6SAlison Schofield * A trace of the calculation per endpoint looks like this:
1628c6ffabc6SAlison Schofield * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1629c6ffabc6SAlison Schofield * pos = 0 * 2 + 0 pos = 0 * 2 + 1
1630c6ffabc6SAlison Schofield * pos: 0 pos: 1
1631c6ffabc6SAlison Schofield *
1632c6ffabc6SAlison Schofield * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1633c6ffabc6SAlison Schofield * pos = 1 * 2 + 0 pos = 1 * 2 + 1
1634c6ffabc6SAlison Schofield * pos: 2 pos = 3
1635c6ffabc6SAlison Schofield *
1636c6ffabc6SAlison Schofield * Note that while this example is simple, the method applies to more
1637c6ffabc6SAlison Schofield * complex topologies, including those with switches.
1638c6ffabc6SAlison Schofield */
1639c6ffabc6SAlison Schofield
1640c6ffabc6SAlison Schofield /* Iterate from endpoint to root_port refining the position */
1641c6ffabc6SAlison Schofield for (iter = port; iter; iter = next_port(iter)) {
1642c6ffabc6SAlison Schofield if (is_cxl_root(iter))
1643c6ffabc6SAlison Schofield break;
1644c6ffabc6SAlison Schofield
1645c6ffabc6SAlison Schofield rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1646c6ffabc6SAlison Schofield if (rc)
1647c6ffabc6SAlison Schofield return rc;
1648c6ffabc6SAlison Schofield
1649c6ffabc6SAlison Schofield pos = pos * parent_ways + parent_pos;
1650c6ffabc6SAlison Schofield }
1651c6ffabc6SAlison Schofield
1652c6ffabc6SAlison Schofield dev_dbg(&cxlmd->dev,
1653c6ffabc6SAlison Schofield "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1654c6ffabc6SAlison Schofield dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1655c6ffabc6SAlison Schofield dev_name(&port->dev), range->start, range->end, pos);
1656c6ffabc6SAlison Schofield
1657c6ffabc6SAlison Schofield return pos;
1658c6ffabc6SAlison Schofield }
1659c6ffabc6SAlison Schofield
cxl_region_sort_targets(struct cxl_region * cxlr)1660a32320b7SDan Williams static int cxl_region_sort_targets(struct cxl_region *cxlr)
1661a32320b7SDan Williams {
1662a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params;
1663a32320b7SDan Williams int i, rc = 0;
1664a32320b7SDan Williams
1665a32320b7SDan Williams for (i = 0; i < p->nr_targets; i++) {
1666a32320b7SDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i];
1667a32320b7SDan Williams
16683cfdfce0SAlison Schofield cxled->pos = cxl_calc_interleave_pos(cxled);
1669a32320b7SDan Williams /*
16703cfdfce0SAlison Schofield * Record that sorting failed, but still continue to calc
16713cfdfce0SAlison Schofield * cxled->pos so that follow-on code paths can reliably
16723cfdfce0SAlison Schofield * do p->targets[cxled->pos] to self-reference their entry.
1673a32320b7SDan Williams */
1674a32320b7SDan Williams if (cxled->pos < 0)
1675a32320b7SDan Williams rc = -ENXIO;
1676a32320b7SDan Williams }
16773cfdfce0SAlison Schofield /* Keep the cxlr target list in interleave position order */
16783cfdfce0SAlison Schofield sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
16793cfdfce0SAlison Schofield cmp_interleave_pos, NULL);
1680a32320b7SDan Williams
1681a32320b7SDan Williams dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1682a32320b7SDan Williams return rc;
1683a32320b7SDan Williams }
1684a32320b7SDan Williams
cxl_region_attach(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)16859995576cSDan Williams static int cxl_region_attach(struct cxl_region *cxlr,
16869995576cSDan Williams struct cxl_endpoint_decoder *cxled, int pos)
16879995576cSDan Williams {
16889995576cSDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
16899995576cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
16909995576cSDan Williams struct cxl_region_params *p = &cxlr->params;
16919995576cSDan Williams struct cxl_port *ep_port, *root_port;
16929995576cSDan Williams struct cxl_dport *dport;
16939995576cSDan Williams int rc = -ENXIO;
16949995576cSDan Williams
16959995576cSDan Williams if (cxled->mode != cxlr->mode) {
16969995576cSDan Williams dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
16979995576cSDan Williams dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
16989995576cSDan Williams return -EINVAL;
16999995576cSDan Williams }
17009995576cSDan Williams
17019995576cSDan Williams if (cxled->mode == CXL_DECODER_DEAD) {
17029995576cSDan Williams dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
17039995576cSDan Williams return -ENODEV;
17049995576cSDan Williams }
17059995576cSDan Williams
17069995576cSDan Williams /* all full of members, or interleave config not established? */
17079995576cSDan Williams if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
17089995576cSDan Williams dev_dbg(&cxlr->dev, "region already active\n");
17099995576cSDan Williams return -EBUSY;
17109995576cSDan Williams } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
17119995576cSDan Williams dev_dbg(&cxlr->dev, "interleave config missing\n");
17129995576cSDan Williams return -ENXIO;
17139995576cSDan Williams }
17149995576cSDan Williams
171507ffcd8eSJim Harris if (p->nr_targets >= p->interleave_ways) {
171607ffcd8eSJim Harris dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
171707ffcd8eSJim Harris p->nr_targets);
171807ffcd8eSJim Harris return -EINVAL;
171907ffcd8eSJim Harris }
172007ffcd8eSJim Harris
1721384e624bSDan Williams ep_port = cxled_to_port(cxled);
1722384e624bSDan Williams root_port = cxlrd_to_port(cxlrd);
1723384e624bSDan Williams dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1724384e624bSDan Williams if (!dport) {
1725384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1726384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1727384e624bSDan Williams dev_name(cxlr->dev.parent));
1728384e624bSDan Williams return -ENXIO;
1729384e624bSDan Williams }
1730384e624bSDan Williams
1731384e624bSDan Williams if (cxled->cxld.target_type != cxlr->type) {
1732384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1733384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1734384e624bSDan Williams cxled->cxld.target_type, cxlr->type);
1735384e624bSDan Williams return -ENXIO;
1736384e624bSDan Williams }
1737384e624bSDan Williams
1738384e624bSDan Williams if (!cxled->dpa_res) {
1739384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1740384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1741384e624bSDan Williams return -ENXIO;
1742384e624bSDan Williams }
1743384e624bSDan Williams
1744384e624bSDan Williams if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1745384e624bSDan Williams resource_size(p->res)) {
1746384e624bSDan Williams dev_dbg(&cxlr->dev,
1747384e624bSDan Williams "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1748384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1749384e624bSDan Williams (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1750384e624bSDan Williams (u64)resource_size(p->res));
1751384e624bSDan Williams return -EINVAL;
1752384e624bSDan Williams }
1753384e624bSDan Williams
1754a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1755a32320b7SDan Williams int i;
1756a32320b7SDan Williams
1757a32320b7SDan Williams rc = cxl_region_attach_auto(cxlr, cxled, pos);
1758384e624bSDan Williams if (rc)
1759a32320b7SDan Williams return rc;
1760a32320b7SDan Williams
1761a32320b7SDan Williams /* await more targets to arrive... */
1762a32320b7SDan Williams if (p->nr_targets < p->interleave_ways)
1763a32320b7SDan Williams return 0;
1764a32320b7SDan Williams
1765a32320b7SDan Williams /*
1766a32320b7SDan Williams * All targets are here, which implies all PCI enumeration that
1767a32320b7SDan Williams * affects this region has been completed. Walk the topology to
1768a32320b7SDan Williams * sort the devices into their relative region decode position.
1769a32320b7SDan Williams */
1770a32320b7SDan Williams rc = cxl_region_sort_targets(cxlr);
1771a32320b7SDan Williams if (rc)
1772a32320b7SDan Williams return rc;
1773a32320b7SDan Williams
1774a32320b7SDan Williams for (i = 0; i < p->nr_targets; i++) {
1775a32320b7SDan Williams cxled = p->targets[i];
1776a32320b7SDan Williams ep_port = cxled_to_port(cxled);
1777a32320b7SDan Williams dport = cxl_find_dport_by_dev(root_port,
1778a32320b7SDan Williams ep_port->host_bridge);
1779a32320b7SDan Williams rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1780a32320b7SDan Williams dport, i);
1781a32320b7SDan Williams if (rc)
1782a32320b7SDan Williams return rc;
1783384e624bSDan Williams }
1784384e624bSDan Williams
1785a32320b7SDan Williams rc = cxl_region_setup_targets(cxlr);
1786a32320b7SDan Williams if (rc)
1787a32320b7SDan Williams return rc;
1788a32320b7SDan Williams
1789a32320b7SDan Williams /*
1790a32320b7SDan Williams * If target setup succeeds in the autodiscovery case
1791a32320b7SDan Williams * then the region is already committed.
1792a32320b7SDan Williams */
1793a32320b7SDan Williams p->state = CXL_CONFIG_COMMIT;
1794a32320b7SDan Williams
1795a32320b7SDan Williams return 0;
1796a32320b7SDan Williams }
1797a32320b7SDan Williams
17989995576cSDan Williams rc = cxl_region_validate_position(cxlr, cxled, pos);
1799b9686e8cSDan Williams if (rc)
18009995576cSDan Williams return rc;
18019995576cSDan Williams
18029995576cSDan Williams rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
18039995576cSDan Williams if (rc)
18049995576cSDan Williams return rc;
1805b9686e8cSDan Williams
1806b9686e8cSDan Williams p->targets[pos] = cxled;
1807b9686e8cSDan Williams cxled->pos = pos;
1808b9686e8cSDan Williams p->nr_targets++;
1809b9686e8cSDan Williams
181027b3f8d1SDan Williams if (p->nr_targets == p->interleave_ways) {
181127b3f8d1SDan Williams rc = cxl_region_setup_targets(cxlr);
181227b3f8d1SDan Williams if (rc)
181307ffcd8eSJim Harris return rc;
1814384e624bSDan Williams p->state = CXL_CONFIG_ACTIVE;
181527b3f8d1SDan Williams }
1816384e624bSDan Williams
18172901c8bdSDan Williams cxled->cxld.interleave_ways = p->interleave_ways;
18182901c8bdSDan Williams cxled->cxld.interleave_granularity = p->interleave_granularity;
1819910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) {
1820910bc55dSDan Williams .start = p->res->start,
1821910bc55dSDan Williams .end = p->res->end,
1822910bc55dSDan Williams };
18232901c8bdSDan Williams
1824c6ffabc6SAlison Schofield if (p->nr_targets != p->interleave_ways)
1825c6ffabc6SAlison Schofield return 0;
1826c6ffabc6SAlison Schofield
1827c6ffabc6SAlison Schofield /*
1828c6ffabc6SAlison Schofield * Test the auto-discovery position calculator function
1829c6ffabc6SAlison Schofield * against this successfully created user-defined region.
1830c6ffabc6SAlison Schofield * A fail message here means that this interleave config
1831c6ffabc6SAlison Schofield * will fail when presented as CXL_REGION_F_AUTO.
1832c6ffabc6SAlison Schofield */
1833c6ffabc6SAlison Schofield for (int i = 0; i < p->nr_targets; i++) {
1834c6ffabc6SAlison Schofield struct cxl_endpoint_decoder *cxled = p->targets[i];
1835c6ffabc6SAlison Schofield int test_pos;
1836c6ffabc6SAlison Schofield
1837c6ffabc6SAlison Schofield test_pos = cxl_calc_interleave_pos(cxled);
1838c6ffabc6SAlison Schofield dev_dbg(&cxled->cxld.dev,
1839c6ffabc6SAlison Schofield "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
1840c6ffabc6SAlison Schofield (test_pos == cxled->pos) ? "success" : "fail",
1841c6ffabc6SAlison Schofield test_pos, cxled->pos);
1842c6ffabc6SAlison Schofield }
1843c6ffabc6SAlison Schofield
1844b9686e8cSDan Williams return 0;
1845b9686e8cSDan Williams }
1846b9686e8cSDan Williams
cxl_region_detach(struct cxl_endpoint_decoder * cxled)1847176baefbSDan Williams static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1848b9686e8cSDan Williams {
1849384e624bSDan Williams struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1850b9686e8cSDan Williams struct cxl_region *cxlr = cxled->cxld.region;
1851b9686e8cSDan Williams struct cxl_region_params *p;
1852176baefbSDan Williams int rc = 0;
1853b9686e8cSDan Williams
1854b9686e8cSDan Williams lockdep_assert_held_write(&cxl_region_rwsem);
1855b9686e8cSDan Williams
1856b9686e8cSDan Williams if (!cxlr)
1857176baefbSDan Williams return 0;
1858b9686e8cSDan Williams
1859b9686e8cSDan Williams p = &cxlr->params;
1860b9686e8cSDan Williams get_device(&cxlr->dev);
1861b9686e8cSDan Williams
1862176baefbSDan Williams if (p->state > CXL_CONFIG_ACTIVE) {
1863176baefbSDan Williams /*
1864176baefbSDan Williams * TODO: tear down all impacted regions if a device is
1865176baefbSDan Williams * removed out of order
1866176baefbSDan Williams */
1867176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1868176baefbSDan Williams if (rc)
1869176baefbSDan Williams goto out;
1870176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE;
1871176baefbSDan Williams }
1872176baefbSDan Williams
1873384e624bSDan Williams for (iter = ep_port; !is_cxl_root(iter);
1874384e624bSDan Williams iter = to_cxl_port(iter->dev.parent))
1875384e624bSDan Williams cxl_port_detach_region(iter, cxlr, cxled);
1876384e624bSDan Williams
1877b9686e8cSDan Williams if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1878b9686e8cSDan Williams p->targets[cxled->pos] != cxled) {
1879b9686e8cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1880b9686e8cSDan Williams
1881b9686e8cSDan Williams dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1882b9686e8cSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1883b9686e8cSDan Williams cxled->pos);
1884b9686e8cSDan Williams goto out;
1885b9686e8cSDan Williams }
1886b9686e8cSDan Williams
188727b3f8d1SDan Williams if (p->state == CXL_CONFIG_ACTIVE) {
1888384e624bSDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
188927b3f8d1SDan Williams cxl_region_teardown_targets(cxlr);
189027b3f8d1SDan Williams }
1891b9686e8cSDan Williams p->targets[cxled->pos] = NULL;
1892b9686e8cSDan Williams p->nr_targets--;
1893910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) {
1894910bc55dSDan Williams .start = 0,
1895910bc55dSDan Williams .end = -1,
1896910bc55dSDan Williams };
1897b9686e8cSDan Williams
1898384e624bSDan Williams /* notify the region driver that one of its targets has departed */
1899b9686e8cSDan Williams up_write(&cxl_region_rwsem);
1900b9686e8cSDan Williams device_release_driver(&cxlr->dev);
1901b9686e8cSDan Williams down_write(&cxl_region_rwsem);
1902b9686e8cSDan Williams out:
1903b9686e8cSDan Williams put_device(&cxlr->dev);
1904176baefbSDan Williams return rc;
1905b9686e8cSDan Williams }
1906b9686e8cSDan Williams
cxl_decoder_kill_region(struct cxl_endpoint_decoder * cxled)1907b9686e8cSDan Williams void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1908b9686e8cSDan Williams {
1909b9686e8cSDan Williams down_write(&cxl_region_rwsem);
1910b9686e8cSDan Williams cxled->mode = CXL_DECODER_DEAD;
1911b9686e8cSDan Williams cxl_region_detach(cxled);
1912b9686e8cSDan Williams up_write(&cxl_region_rwsem);
1913b9686e8cSDan Williams }
1914b9686e8cSDan Williams
attach_target(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos,unsigned int state)19153528b1e1SDan Williams static int attach_target(struct cxl_region *cxlr,
19163528b1e1SDan Williams struct cxl_endpoint_decoder *cxled, int pos,
19173528b1e1SDan Williams unsigned int state)
1918b9686e8cSDan Williams {
19193528b1e1SDan Williams int rc = 0;
1920b9686e8cSDan Williams
19213528b1e1SDan Williams if (state == TASK_INTERRUPTIBLE)
1922b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem);
19233528b1e1SDan Williams else
19243528b1e1SDan Williams down_write(&cxl_region_rwsem);
1925b9686e8cSDan Williams if (rc)
19263528b1e1SDan Williams return rc;
19273528b1e1SDan Williams
1928b9686e8cSDan Williams down_read(&cxl_dpa_rwsem);
19293528b1e1SDan Williams rc = cxl_region_attach(cxlr, cxled, pos);
1930b9686e8cSDan Williams up_read(&cxl_dpa_rwsem);
1931b9686e8cSDan Williams up_write(&cxl_region_rwsem);
1932b9686e8cSDan Williams return rc;
1933b9686e8cSDan Williams }
1934b9686e8cSDan Williams
detach_target(struct cxl_region * cxlr,int pos)1935b9686e8cSDan Williams static int detach_target(struct cxl_region *cxlr, int pos)
1936b9686e8cSDan Williams {
1937b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params;
1938b9686e8cSDan Williams int rc;
1939b9686e8cSDan Williams
1940b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem);
1941b9686e8cSDan Williams if (rc)
1942b9686e8cSDan Williams return rc;
1943b9686e8cSDan Williams
1944b9686e8cSDan Williams if (pos >= p->interleave_ways) {
1945b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1946b9686e8cSDan Williams p->interleave_ways);
1947b9686e8cSDan Williams rc = -ENXIO;
1948b9686e8cSDan Williams goto out;
1949b9686e8cSDan Williams }
1950b9686e8cSDan Williams
1951b9686e8cSDan Williams if (!p->targets[pos]) {
1952b9686e8cSDan Williams rc = 0;
1953b9686e8cSDan Williams goto out;
1954b9686e8cSDan Williams }
1955b9686e8cSDan Williams
1956176baefbSDan Williams rc = cxl_region_detach(p->targets[pos]);
1957b9686e8cSDan Williams out:
1958b9686e8cSDan Williams up_write(&cxl_region_rwsem);
1959b9686e8cSDan Williams return rc;
1960b9686e8cSDan Williams }
1961b9686e8cSDan Williams
store_targetN(struct cxl_region * cxlr,const char * buf,int pos,size_t len)1962b9686e8cSDan Williams static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1963b9686e8cSDan Williams size_t len)
1964b9686e8cSDan Williams {
1965b9686e8cSDan Williams int rc;
1966b9686e8cSDan Williams
1967b9686e8cSDan Williams if (sysfs_streq(buf, "\n"))
1968b9686e8cSDan Williams rc = detach_target(cxlr, pos);
19693528b1e1SDan Williams else {
19703528b1e1SDan Williams struct device *dev;
19713528b1e1SDan Williams
19723528b1e1SDan Williams dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
19733528b1e1SDan Williams if (!dev)
19743528b1e1SDan Williams return -ENODEV;
19753528b1e1SDan Williams
19763528b1e1SDan Williams if (!is_endpoint_decoder(dev)) {
19773528b1e1SDan Williams rc = -EINVAL;
19783528b1e1SDan Williams goto out;
19793528b1e1SDan Williams }
19803528b1e1SDan Williams
19813528b1e1SDan Williams rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
19823528b1e1SDan Williams TASK_INTERRUPTIBLE);
19833528b1e1SDan Williams out:
19843528b1e1SDan Williams put_device(dev);
19853528b1e1SDan Williams }
1986b9686e8cSDan Williams
1987b9686e8cSDan Williams if (rc < 0)
1988b9686e8cSDan Williams return rc;
1989b9686e8cSDan Williams return len;
1990b9686e8cSDan Williams }
1991b9686e8cSDan Williams
1992b9686e8cSDan Williams #define TARGET_ATTR_RW(n) \
1993b9686e8cSDan Williams static ssize_t target##n##_show( \
1994b9686e8cSDan Williams struct device *dev, struct device_attribute *attr, char *buf) \
1995b9686e8cSDan Williams { \
1996b9686e8cSDan Williams return show_targetN(to_cxl_region(dev), buf, (n)); \
1997b9686e8cSDan Williams } \
1998b9686e8cSDan Williams static ssize_t target##n##_store(struct device *dev, \
1999b9686e8cSDan Williams struct device_attribute *attr, \
2000b9686e8cSDan Williams const char *buf, size_t len) \
2001b9686e8cSDan Williams { \
2002b9686e8cSDan Williams return store_targetN(to_cxl_region(dev), buf, (n), len); \
2003b9686e8cSDan Williams } \
2004b9686e8cSDan Williams static DEVICE_ATTR_RW(target##n)
2005b9686e8cSDan Williams
2006b9686e8cSDan Williams TARGET_ATTR_RW(0);
2007b9686e8cSDan Williams TARGET_ATTR_RW(1);
2008b9686e8cSDan Williams TARGET_ATTR_RW(2);
2009b9686e8cSDan Williams TARGET_ATTR_RW(3);
2010b9686e8cSDan Williams TARGET_ATTR_RW(4);
2011b9686e8cSDan Williams TARGET_ATTR_RW(5);
2012b9686e8cSDan Williams TARGET_ATTR_RW(6);
2013b9686e8cSDan Williams TARGET_ATTR_RW(7);
2014b9686e8cSDan Williams TARGET_ATTR_RW(8);
2015b9686e8cSDan Williams TARGET_ATTR_RW(9);
2016b9686e8cSDan Williams TARGET_ATTR_RW(10);
2017b9686e8cSDan Williams TARGET_ATTR_RW(11);
2018b9686e8cSDan Williams TARGET_ATTR_RW(12);
2019b9686e8cSDan Williams TARGET_ATTR_RW(13);
2020b9686e8cSDan Williams TARGET_ATTR_RW(14);
2021b9686e8cSDan Williams TARGET_ATTR_RW(15);
2022b9686e8cSDan Williams
2023b9686e8cSDan Williams static struct attribute *target_attrs[] = {
2024b9686e8cSDan Williams &dev_attr_target0.attr,
2025b9686e8cSDan Williams &dev_attr_target1.attr,
2026b9686e8cSDan Williams &dev_attr_target2.attr,
2027b9686e8cSDan Williams &dev_attr_target3.attr,
2028b9686e8cSDan Williams &dev_attr_target4.attr,
2029b9686e8cSDan Williams &dev_attr_target5.attr,
2030b9686e8cSDan Williams &dev_attr_target6.attr,
2031b9686e8cSDan Williams &dev_attr_target7.attr,
2032b9686e8cSDan Williams &dev_attr_target8.attr,
2033b9686e8cSDan Williams &dev_attr_target9.attr,
2034b9686e8cSDan Williams &dev_attr_target10.attr,
2035b9686e8cSDan Williams &dev_attr_target11.attr,
2036b9686e8cSDan Williams &dev_attr_target12.attr,
2037b9686e8cSDan Williams &dev_attr_target13.attr,
2038b9686e8cSDan Williams &dev_attr_target14.attr,
2039b9686e8cSDan Williams &dev_attr_target15.attr,
2040b9686e8cSDan Williams NULL,
2041b9686e8cSDan Williams };
2042b9686e8cSDan Williams
cxl_region_target_visible(struct kobject * kobj,struct attribute * a,int n)2043b9686e8cSDan Williams static umode_t cxl_region_target_visible(struct kobject *kobj,
2044b9686e8cSDan Williams struct attribute *a, int n)
2045b9686e8cSDan Williams {
2046b9686e8cSDan Williams struct device *dev = kobj_to_dev(kobj);
2047b9686e8cSDan Williams struct cxl_region *cxlr = to_cxl_region(dev);
2048b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params;
2049b9686e8cSDan Williams
2050b9686e8cSDan Williams if (n < p->interleave_ways)
2051b9686e8cSDan Williams return a->mode;
2052b9686e8cSDan Williams return 0;
2053b9686e8cSDan Williams }
2054b9686e8cSDan Williams
2055b9686e8cSDan Williams static const struct attribute_group cxl_region_target_group = {
2056b9686e8cSDan Williams .attrs = target_attrs,
2057b9686e8cSDan Williams .is_visible = cxl_region_target_visible,
2058b9686e8cSDan Williams };
2059b9686e8cSDan Williams
get_cxl_region_target_group(void)2060b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void)
2061b9686e8cSDan Williams {
2062b9686e8cSDan Williams return &cxl_region_target_group;
2063b9686e8cSDan Williams }
2064b9686e8cSDan Williams
2065dd5ba0ebSBen Widawsky static const struct attribute_group *region_groups[] = {
2066dd5ba0ebSBen Widawsky &cxl_base_attribute_group,
2067dd5ba0ebSBen Widawsky &cxl_region_group,
2068b9686e8cSDan Williams &cxl_region_target_group,
2069dd5ba0ebSBen Widawsky NULL,
2070dd5ba0ebSBen Widawsky };
2071dd5ba0ebSBen Widawsky
cxl_region_release(struct device * dev)2072779dd20cSBen Widawsky static void cxl_region_release(struct device *dev)
2073779dd20cSBen Widawsky {
20748f401ec1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2075779dd20cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev);
20768f401ec1SDan Williams int id = atomic_read(&cxlrd->region_id);
20778f401ec1SDan Williams
20788f401ec1SDan Williams /*
20798f401ec1SDan Williams * Try to reuse the recently idled id rather than the cached
20808f401ec1SDan Williams * next id to prevent the region id space from increasing
20818f401ec1SDan Williams * unnecessarily.
20828f401ec1SDan Williams */
20838f401ec1SDan Williams if (cxlr->id < id)
20848f401ec1SDan Williams if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
20858f401ec1SDan Williams memregion_free(id);
20868f401ec1SDan Williams goto out;
20878f401ec1SDan Williams }
2088779dd20cSBen Widawsky
2089779dd20cSBen Widawsky memregion_free(cxlr->id);
20908f401ec1SDan Williams out:
20918f401ec1SDan Williams put_device(dev->parent);
2092779dd20cSBen Widawsky kfree(cxlr);
2093779dd20cSBen Widawsky }
2094779dd20cSBen Widawsky
20958d48817dSDan Williams const struct device_type cxl_region_type = {
2096779dd20cSBen Widawsky .name = "cxl_region",
2097779dd20cSBen Widawsky .release = cxl_region_release,
2098dd5ba0ebSBen Widawsky .groups = region_groups
2099779dd20cSBen Widawsky };
2100779dd20cSBen Widawsky
is_cxl_region(struct device * dev)2101779dd20cSBen Widawsky bool is_cxl_region(struct device *dev)
2102779dd20cSBen Widawsky {
2103779dd20cSBen Widawsky return dev->type == &cxl_region_type;
2104779dd20cSBen Widawsky }
2105779dd20cSBen Widawsky EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2106779dd20cSBen Widawsky
to_cxl_region(struct device * dev)2107779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev)
2108779dd20cSBen Widawsky {
2109779dd20cSBen Widawsky if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2110779dd20cSBen Widawsky "not a cxl_region device\n"))
2111779dd20cSBen Widawsky return NULL;
2112779dd20cSBen Widawsky
2113779dd20cSBen Widawsky return container_of(dev, struct cxl_region, dev);
2114779dd20cSBen Widawsky }
2115779dd20cSBen Widawsky
unregister_region(void * dev)2116779dd20cSBen Widawsky static void unregister_region(void *dev)
2117779dd20cSBen Widawsky {
211823a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev);
21190d9e7340SDan Williams struct cxl_region_params *p = &cxlr->params;
21200d9e7340SDan Williams int i;
212123a22cd1SDan Williams
212223a22cd1SDan Williams device_del(dev);
21230d9e7340SDan Williams
21240d9e7340SDan Williams /*
21250d9e7340SDan Williams * Now that region sysfs is shutdown, the parameter block is now
21260d9e7340SDan Williams * read-only, so no need to hold the region rwsem to access the
21270d9e7340SDan Williams * region parameters.
21280d9e7340SDan Williams */
21290d9e7340SDan Williams for (i = 0; i < p->interleave_ways; i++)
21300d9e7340SDan Williams detach_target(cxlr, i);
21310d9e7340SDan Williams
213223a22cd1SDan Williams cxl_region_iomem_release(cxlr);
213323a22cd1SDan Williams put_device(dev);
2134779dd20cSBen Widawsky }
2135779dd20cSBen Widawsky
2136779dd20cSBen Widawsky static struct lock_class_key cxl_region_key;
2137779dd20cSBen Widawsky
cxl_region_alloc(struct cxl_root_decoder * cxlrd,int id)2138779dd20cSBen Widawsky static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2139779dd20cSBen Widawsky {
2140779dd20cSBen Widawsky struct cxl_region *cxlr;
2141779dd20cSBen Widawsky struct device *dev;
2142779dd20cSBen Widawsky
2143779dd20cSBen Widawsky cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2144779dd20cSBen Widawsky if (!cxlr) {
2145779dd20cSBen Widawsky memregion_free(id);
2146779dd20cSBen Widawsky return ERR_PTR(-ENOMEM);
2147779dd20cSBen Widawsky }
2148779dd20cSBen Widawsky
2149779dd20cSBen Widawsky dev = &cxlr->dev;
2150779dd20cSBen Widawsky device_initialize(dev);
2151779dd20cSBen Widawsky lockdep_set_class(&dev->mutex, &cxl_region_key);
2152779dd20cSBen Widawsky dev->parent = &cxlrd->cxlsd.cxld.dev;
21538f401ec1SDan Williams /*
21548f401ec1SDan Williams * Keep root decoder pinned through cxl_region_release to fixup
21558f401ec1SDan Williams * region id allocations
21568f401ec1SDan Williams */
21578f401ec1SDan Williams get_device(dev->parent);
2158779dd20cSBen Widawsky device_set_pm_not_required(dev);
2159779dd20cSBen Widawsky dev->bus = &cxl_bus_type;
2160779dd20cSBen Widawsky dev->type = &cxl_region_type;
2161779dd20cSBen Widawsky cxlr->id = id;
2162779dd20cSBen Widawsky
2163779dd20cSBen Widawsky return cxlr;
2164779dd20cSBen Widawsky }
2165779dd20cSBen Widawsky
2166779dd20cSBen Widawsky /**
2167779dd20cSBen Widawsky * devm_cxl_add_region - Adds a region to a decoder
2168779dd20cSBen Widawsky * @cxlrd: root decoder
2169779dd20cSBen Widawsky * @id: memregion id to create, or memregion_free() on failure
2170779dd20cSBen Widawsky * @mode: mode for the endpoint decoders of this region
2171779dd20cSBen Widawsky * @type: select whether this is an expander or accelerator (type-2 or type-3)
2172779dd20cSBen Widawsky *
2173779dd20cSBen Widawsky * This is the second step of region initialization. Regions exist within an
2174779dd20cSBen Widawsky * address space which is mapped by a @cxlrd.
2175779dd20cSBen Widawsky *
2176779dd20cSBen Widawsky * Return: 0 if the region was added to the @cxlrd, else returns negative error
2177779dd20cSBen Widawsky * code. The region will be named "regionZ" where Z is the unique region number.
2178779dd20cSBen Widawsky */
devm_cxl_add_region(struct cxl_root_decoder * cxlrd,int id,enum cxl_decoder_mode mode,enum cxl_decoder_type type)2179779dd20cSBen Widawsky static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2180779dd20cSBen Widawsky int id,
2181779dd20cSBen Widawsky enum cxl_decoder_mode mode,
2182779dd20cSBen Widawsky enum cxl_decoder_type type)
2183779dd20cSBen Widawsky {
2184779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2185779dd20cSBen Widawsky struct cxl_region *cxlr;
2186779dd20cSBen Widawsky struct device *dev;
2187779dd20cSBen Widawsky int rc;
2188779dd20cSBen Widawsky
2189779dd20cSBen Widawsky cxlr = cxl_region_alloc(cxlrd, id);
2190779dd20cSBen Widawsky if (IS_ERR(cxlr))
2191779dd20cSBen Widawsky return cxlr;
2192779dd20cSBen Widawsky cxlr->mode = mode;
2193779dd20cSBen Widawsky cxlr->type = type;
2194779dd20cSBen Widawsky
2195779dd20cSBen Widawsky dev = &cxlr->dev;
2196779dd20cSBen Widawsky rc = dev_set_name(dev, "region%d", id);
2197779dd20cSBen Widawsky if (rc)
2198779dd20cSBen Widawsky goto err;
2199779dd20cSBen Widawsky
2200779dd20cSBen Widawsky rc = device_add(dev);
2201779dd20cSBen Widawsky if (rc)
2202779dd20cSBen Widawsky goto err;
2203779dd20cSBen Widawsky
22047481653dSDan Williams rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2205779dd20cSBen Widawsky if (rc)
2206779dd20cSBen Widawsky return ERR_PTR(rc);
2207779dd20cSBen Widawsky
22087481653dSDan Williams dev_dbg(port->uport_dev, "%s: created %s\n",
2209779dd20cSBen Widawsky dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2210779dd20cSBen Widawsky return cxlr;
2211779dd20cSBen Widawsky
2212779dd20cSBen Widawsky err:
2213779dd20cSBen Widawsky put_device(dev);
2214779dd20cSBen Widawsky return ERR_PTR(rc);
2215779dd20cSBen Widawsky }
2216779dd20cSBen Widawsky
__create_region_show(struct cxl_root_decoder * cxlrd,char * buf)22176e099264SDan Williams static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
22186e099264SDan Williams {
22196e099264SDan Williams return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
22206e099264SDan Williams }
22216e099264SDan Williams
create_pmem_region_show(struct device * dev,struct device_attribute * attr,char * buf)2222779dd20cSBen Widawsky static ssize_t create_pmem_region_show(struct device *dev,
2223779dd20cSBen Widawsky struct device_attribute *attr, char *buf)
2224779dd20cSBen Widawsky {
22256e099264SDan Williams return __create_region_show(to_cxl_root_decoder(dev), buf);
22266e099264SDan Williams }
2227779dd20cSBen Widawsky
create_ram_region_show(struct device * dev,struct device_attribute * attr,char * buf)22286e099264SDan Williams static ssize_t create_ram_region_show(struct device *dev,
22296e099264SDan Williams struct device_attribute *attr, char *buf)
22306e099264SDan Williams {
22316e099264SDan Williams return __create_region_show(to_cxl_root_decoder(dev), buf);
22326e099264SDan Williams }
22336e099264SDan Williams
__create_region(struct cxl_root_decoder * cxlrd,enum cxl_decoder_mode mode,int id)22346e099264SDan Williams static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
22356e099264SDan Williams enum cxl_decoder_mode mode, int id)
22366e099264SDan Williams {
22376e099264SDan Williams int rc;
22386e099264SDan Williams
2239d8316838SLi Zhijian switch (mode) {
2240d8316838SLi Zhijian case CXL_DECODER_RAM:
2241d8316838SLi Zhijian case CXL_DECODER_PMEM:
2242d8316838SLi Zhijian break;
2243d8316838SLi Zhijian default:
2244d8316838SLi Zhijian dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2245d8316838SLi Zhijian return ERR_PTR(-EINVAL);
2246d8316838SLi Zhijian }
2247d8316838SLi Zhijian
22486e099264SDan Williams rc = memregion_alloc(GFP_KERNEL);
22496e099264SDan Williams if (rc < 0)
22506e099264SDan Williams return ERR_PTR(rc);
22516e099264SDan Williams
22526e099264SDan Williams if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
22536e099264SDan Williams memregion_free(rc);
22546e099264SDan Williams return ERR_PTR(-EBUSY);
22556e099264SDan Williams }
22566e099264SDan Williams
22575aa39a91SDan Williams return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2258779dd20cSBen Widawsky }
2259779dd20cSBen Widawsky
create_pmem_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2260779dd20cSBen Widawsky static ssize_t create_pmem_region_store(struct device *dev,
2261779dd20cSBen Widawsky struct device_attribute *attr,
2262779dd20cSBen Widawsky const char *buf, size_t len)
2263779dd20cSBen Widawsky {
2264779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2265779dd20cSBen Widawsky struct cxl_region *cxlr;
22666e099264SDan Williams int rc, id;
2267779dd20cSBen Widawsky
2268779dd20cSBen Widawsky rc = sscanf(buf, "region%d\n", &id);
2269779dd20cSBen Widawsky if (rc != 1)
2270779dd20cSBen Widawsky return -EINVAL;
2271779dd20cSBen Widawsky
22726e099264SDan Williams cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2273779dd20cSBen Widawsky if (IS_ERR(cxlr))
2274779dd20cSBen Widawsky return PTR_ERR(cxlr);
2275779dd20cSBen Widawsky
2276779dd20cSBen Widawsky return len;
2277779dd20cSBen Widawsky }
2278779dd20cSBen Widawsky DEVICE_ATTR_RW(create_pmem_region);
2279779dd20cSBen Widawsky
create_ram_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)22806e099264SDan Williams static ssize_t create_ram_region_store(struct device *dev,
22816e099264SDan Williams struct device_attribute *attr,
22826e099264SDan Williams const char *buf, size_t len)
22836e099264SDan Williams {
22846e099264SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
22856e099264SDan Williams struct cxl_region *cxlr;
22866e099264SDan Williams int rc, id;
22876e099264SDan Williams
22886e099264SDan Williams rc = sscanf(buf, "region%d\n", &id);
22896e099264SDan Williams if (rc != 1)
22906e099264SDan Williams return -EINVAL;
22916e099264SDan Williams
22926e099264SDan Williams cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
22936e099264SDan Williams if (IS_ERR(cxlr))
22946e099264SDan Williams return PTR_ERR(cxlr);
22956e099264SDan Williams
22966e099264SDan Williams return len;
22976e099264SDan Williams }
22986e099264SDan Williams DEVICE_ATTR_RW(create_ram_region);
22996e099264SDan Williams
region_show(struct device * dev,struct device_attribute * attr,char * buf)2300b9686e8cSDan Williams static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2301b9686e8cSDan Williams char *buf)
2302b9686e8cSDan Williams {
2303b9686e8cSDan Williams struct cxl_decoder *cxld = to_cxl_decoder(dev);
2304b9686e8cSDan Williams ssize_t rc;
2305b9686e8cSDan Williams
2306b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem);
2307b9686e8cSDan Williams if (rc)
2308b9686e8cSDan Williams return rc;
2309b9686e8cSDan Williams
2310b9686e8cSDan Williams if (cxld->region)
2311b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2312b9686e8cSDan Williams else
2313b9686e8cSDan Williams rc = sysfs_emit(buf, "\n");
2314b9686e8cSDan Williams up_read(&cxl_region_rwsem);
2315b9686e8cSDan Williams
2316b9686e8cSDan Williams return rc;
2317b9686e8cSDan Williams }
2318b9686e8cSDan Williams DEVICE_ATTR_RO(region);
2319b9686e8cSDan Williams
2320779dd20cSBen Widawsky static struct cxl_region *
cxl_find_region_by_name(struct cxl_root_decoder * cxlrd,const char * name)2321779dd20cSBen Widawsky cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2322779dd20cSBen Widawsky {
2323779dd20cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2324779dd20cSBen Widawsky struct device *region_dev;
2325779dd20cSBen Widawsky
2326779dd20cSBen Widawsky region_dev = device_find_child_by_name(&cxld->dev, name);
2327779dd20cSBen Widawsky if (!region_dev)
2328779dd20cSBen Widawsky return ERR_PTR(-ENODEV);
2329779dd20cSBen Widawsky
2330779dd20cSBen Widawsky return to_cxl_region(region_dev);
2331779dd20cSBen Widawsky }
2332779dd20cSBen Widawsky
delete_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2333779dd20cSBen Widawsky static ssize_t delete_region_store(struct device *dev,
2334779dd20cSBen Widawsky struct device_attribute *attr,
2335779dd20cSBen Widawsky const char *buf, size_t len)
2336779dd20cSBen Widawsky {
2337779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2338779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(dev->parent);
2339779dd20cSBen Widawsky struct cxl_region *cxlr;
2340779dd20cSBen Widawsky
2341779dd20cSBen Widawsky cxlr = cxl_find_region_by_name(cxlrd, buf);
2342779dd20cSBen Widawsky if (IS_ERR(cxlr))
2343779dd20cSBen Widawsky return PTR_ERR(cxlr);
2344779dd20cSBen Widawsky
23457481653dSDan Williams devm_release_action(port->uport_dev, unregister_region, cxlr);
2346779dd20cSBen Widawsky put_device(&cxlr->dev);
2347779dd20cSBen Widawsky
2348779dd20cSBen Widawsky return len;
2349779dd20cSBen Widawsky }
2350779dd20cSBen Widawsky DEVICE_ATTR_WO(delete_region);
235123a22cd1SDan Williams
cxl_pmem_region_release(struct device * dev)235204ad63f0SDan Williams static void cxl_pmem_region_release(struct device *dev)
235304ad63f0SDan Williams {
235404ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
235504ad63f0SDan Williams int i;
235604ad63f0SDan Williams
235704ad63f0SDan Williams for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
235804ad63f0SDan Williams struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
235904ad63f0SDan Williams
236004ad63f0SDan Williams put_device(&cxlmd->dev);
236104ad63f0SDan Williams }
236204ad63f0SDan Williams
236304ad63f0SDan Williams kfree(cxlr_pmem);
236404ad63f0SDan Williams }
236504ad63f0SDan Williams
236604ad63f0SDan Williams static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
236704ad63f0SDan Williams &cxl_base_attribute_group,
236804ad63f0SDan Williams NULL,
236904ad63f0SDan Williams };
237004ad63f0SDan Williams
237104ad63f0SDan Williams const struct device_type cxl_pmem_region_type = {
237204ad63f0SDan Williams .name = "cxl_pmem_region",
237304ad63f0SDan Williams .release = cxl_pmem_region_release,
237404ad63f0SDan Williams .groups = cxl_pmem_region_attribute_groups,
237504ad63f0SDan Williams };
237604ad63f0SDan Williams
is_cxl_pmem_region(struct device * dev)237704ad63f0SDan Williams bool is_cxl_pmem_region(struct device *dev)
237804ad63f0SDan Williams {
237904ad63f0SDan Williams return dev->type == &cxl_pmem_region_type;
238004ad63f0SDan Williams }
238104ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
238204ad63f0SDan Williams
to_cxl_pmem_region(struct device * dev)238304ad63f0SDan Williams struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
238404ad63f0SDan Williams {
238504ad63f0SDan Williams if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
238604ad63f0SDan Williams "not a cxl_pmem_region device\n"))
238704ad63f0SDan Williams return NULL;
238804ad63f0SDan Williams return container_of(dev, struct cxl_pmem_region, dev);
238904ad63f0SDan Williams }
239004ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
239104ad63f0SDan Williams
2392f0832a58SAlison Schofield struct cxl_poison_context {
2393f0832a58SAlison Schofield struct cxl_port *port;
2394f0832a58SAlison Schofield enum cxl_decoder_mode mode;
2395f0832a58SAlison Schofield u64 offset;
2396f0832a58SAlison Schofield };
2397f0832a58SAlison Schofield
cxl_get_poison_unmapped(struct cxl_memdev * cxlmd,struct cxl_poison_context * ctx)2398f0832a58SAlison Schofield static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2399f0832a58SAlison Schofield struct cxl_poison_context *ctx)
2400f0832a58SAlison Schofield {
2401f0832a58SAlison Schofield struct cxl_dev_state *cxlds = cxlmd->cxlds;
2402f0832a58SAlison Schofield u64 offset, length;
2403f0832a58SAlison Schofield int rc = 0;
2404f0832a58SAlison Schofield
2405f0832a58SAlison Schofield /*
2406f0832a58SAlison Schofield * Collect poison for the remaining unmapped resources
2407f0832a58SAlison Schofield * after poison is collected by committed endpoints.
2408f0832a58SAlison Schofield *
2409f0832a58SAlison Schofield * Knowing that PMEM must always follow RAM, get poison
2410f0832a58SAlison Schofield * for unmapped resources based on the last decoder's mode:
2411f0832a58SAlison Schofield * ram: scan remains of ram range, then any pmem range
2412f0832a58SAlison Schofield * pmem: scan remains of pmem range
2413f0832a58SAlison Schofield */
2414f0832a58SAlison Schofield
2415f0832a58SAlison Schofield if (ctx->mode == CXL_DECODER_RAM) {
2416f0832a58SAlison Schofield offset = ctx->offset;
2417f0832a58SAlison Schofield length = resource_size(&cxlds->ram_res) - offset;
2418f0832a58SAlison Schofield rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2419f0832a58SAlison Schofield if (rc == -EFAULT)
2420f0832a58SAlison Schofield rc = 0;
2421f0832a58SAlison Schofield if (rc)
2422f0832a58SAlison Schofield return rc;
2423f0832a58SAlison Schofield }
2424f0832a58SAlison Schofield if (ctx->mode == CXL_DECODER_PMEM) {
2425f0832a58SAlison Schofield offset = ctx->offset;
2426f0832a58SAlison Schofield length = resource_size(&cxlds->dpa_res) - offset;
2427f0832a58SAlison Schofield if (!length)
2428f0832a58SAlison Schofield return 0;
2429f0832a58SAlison Schofield } else if (resource_size(&cxlds->pmem_res)) {
2430f0832a58SAlison Schofield offset = cxlds->pmem_res.start;
2431f0832a58SAlison Schofield length = resource_size(&cxlds->pmem_res);
2432f0832a58SAlison Schofield } else {
2433f0832a58SAlison Schofield return 0;
2434f0832a58SAlison Schofield }
2435f0832a58SAlison Schofield
2436f0832a58SAlison Schofield return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2437f0832a58SAlison Schofield }
2438f0832a58SAlison Schofield
poison_by_decoder(struct device * dev,void * arg)2439f0832a58SAlison Schofield static int poison_by_decoder(struct device *dev, void *arg)
2440f0832a58SAlison Schofield {
2441f0832a58SAlison Schofield struct cxl_poison_context *ctx = arg;
2442f0832a58SAlison Schofield struct cxl_endpoint_decoder *cxled;
2443f0832a58SAlison Schofield struct cxl_memdev *cxlmd;
2444f0832a58SAlison Schofield u64 offset, length;
2445f0832a58SAlison Schofield int rc = 0;
2446f0832a58SAlison Schofield
2447f0832a58SAlison Schofield if (!is_endpoint_decoder(dev))
2448f0832a58SAlison Schofield return rc;
2449f0832a58SAlison Schofield
2450f0832a58SAlison Schofield cxled = to_cxl_endpoint_decoder(dev);
2451f0832a58SAlison Schofield if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2452f0832a58SAlison Schofield return rc;
2453f0832a58SAlison Schofield
2454f0832a58SAlison Schofield /*
2455f0832a58SAlison Schofield * Regions are only created with single mode decoders: pmem or ram.
2456f0832a58SAlison Schofield * Linux does not support mixed mode decoders. This means that
2457f0832a58SAlison Schofield * reading poison per endpoint decoder adheres to the requirement
2458f0832a58SAlison Schofield * that poison reads of pmem and ram must be separated.
2459f0832a58SAlison Schofield * CXL 3.0 Spec 8.2.9.8.4.1
2460f0832a58SAlison Schofield */
2461f0832a58SAlison Schofield if (cxled->mode == CXL_DECODER_MIXED) {
2462f0832a58SAlison Schofield dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2463f0832a58SAlison Schofield return rc;
2464f0832a58SAlison Schofield }
2465f0832a58SAlison Schofield
2466f0832a58SAlison Schofield cxlmd = cxled_to_memdev(cxled);
2467f0832a58SAlison Schofield if (cxled->skip) {
2468f0832a58SAlison Schofield offset = cxled->dpa_res->start - cxled->skip;
2469f0832a58SAlison Schofield length = cxled->skip;
2470f0832a58SAlison Schofield rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2471f0832a58SAlison Schofield if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2472f0832a58SAlison Schofield rc = 0;
2473f0832a58SAlison Schofield if (rc)
2474f0832a58SAlison Schofield return rc;
2475f0832a58SAlison Schofield }
2476f0832a58SAlison Schofield
2477f0832a58SAlison Schofield offset = cxled->dpa_res->start;
2478f0832a58SAlison Schofield length = cxled->dpa_res->end - offset + 1;
2479f0832a58SAlison Schofield rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2480f0832a58SAlison Schofield if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2481f0832a58SAlison Schofield rc = 0;
2482f0832a58SAlison Schofield if (rc)
2483f0832a58SAlison Schofield return rc;
2484f0832a58SAlison Schofield
2485f0832a58SAlison Schofield /* Iterate until commit_end is reached */
2486f0832a58SAlison Schofield if (cxled->cxld.id == ctx->port->commit_end) {
2487f0832a58SAlison Schofield ctx->offset = cxled->dpa_res->end + 1;
2488f0832a58SAlison Schofield ctx->mode = cxled->mode;
2489f0832a58SAlison Schofield return 1;
2490f0832a58SAlison Schofield }
2491f0832a58SAlison Schofield
2492f0832a58SAlison Schofield return 0;
2493f0832a58SAlison Schofield }
2494f0832a58SAlison Schofield
cxl_get_poison_by_endpoint(struct cxl_port * port)2495f0832a58SAlison Schofield int cxl_get_poison_by_endpoint(struct cxl_port *port)
2496f0832a58SAlison Schofield {
2497f0832a58SAlison Schofield struct cxl_poison_context ctx;
2498f0832a58SAlison Schofield int rc = 0;
2499f0832a58SAlison Schofield
2500f0832a58SAlison Schofield ctx = (struct cxl_poison_context) {
2501f0832a58SAlison Schofield .port = port
2502f0832a58SAlison Schofield };
2503f0832a58SAlison Schofield
2504f0832a58SAlison Schofield rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2505f0832a58SAlison Schofield if (rc == 1)
25067481653dSDan Williams rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
25077481653dSDan Williams &ctx);
2508f0832a58SAlison Schofield
2509f0832a58SAlison Schofield return rc;
2510f0832a58SAlison Schofield }
2511f0832a58SAlison Schofield
251204ad63f0SDan Williams static struct lock_class_key cxl_pmem_region_key;
251304ad63f0SDan Williams
cxl_pmem_region_alloc(struct cxl_region * cxlr)251404ad63f0SDan Williams static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
251504ad63f0SDan Williams {
251604ad63f0SDan Williams struct cxl_region_params *p = &cxlr->params;
2517f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb;
251804ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem;
251904ad63f0SDan Williams struct device *dev;
252004ad63f0SDan Williams int i;
252104ad63f0SDan Williams
252204ad63f0SDan Williams down_read(&cxl_region_rwsem);
252304ad63f0SDan Williams if (p->state != CXL_CONFIG_COMMIT) {
252404ad63f0SDan Williams cxlr_pmem = ERR_PTR(-ENXIO);
252504ad63f0SDan Williams goto out;
252604ad63f0SDan Williams }
252704ad63f0SDan Williams
252804ad63f0SDan Williams cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
252904ad63f0SDan Williams GFP_KERNEL);
253004ad63f0SDan Williams if (!cxlr_pmem) {
253104ad63f0SDan Williams cxlr_pmem = ERR_PTR(-ENOMEM);
253204ad63f0SDan Williams goto out;
253304ad63f0SDan Williams }
253404ad63f0SDan Williams
253504ad63f0SDan Williams cxlr_pmem->hpa_range.start = p->res->start;
253604ad63f0SDan Williams cxlr_pmem->hpa_range.end = p->res->end;
253704ad63f0SDan Williams
253804ad63f0SDan Williams /* Snapshot the region configuration underneath the cxl_region_rwsem */
253904ad63f0SDan Williams cxlr_pmem->nr_mappings = p->nr_targets;
254004ad63f0SDan Williams for (i = 0; i < p->nr_targets; i++) {
254104ad63f0SDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i];
254204ad63f0SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
254304ad63f0SDan Williams struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
254404ad63f0SDan Williams
2545f17b558dSDan Williams /*
2546f17b558dSDan Williams * Regions never span CXL root devices, so by definition the
2547f17b558dSDan Williams * bridge for one device is the same for all.
2548f17b558dSDan Williams */
2549f17b558dSDan Williams if (i == 0) {
2550d35b495dSDan Williams cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2551f17b558dSDan Williams if (!cxl_nvb) {
255224b9362cSLi Zhijian kfree(cxlr_pmem);
2553f17b558dSDan Williams cxlr_pmem = ERR_PTR(-ENODEV);
2554f17b558dSDan Williams goto out;
2555f17b558dSDan Williams }
2556f17b558dSDan Williams cxlr->cxl_nvb = cxl_nvb;
2557f17b558dSDan Williams }
255804ad63f0SDan Williams m->cxlmd = cxlmd;
255904ad63f0SDan Williams get_device(&cxlmd->dev);
256004ad63f0SDan Williams m->start = cxled->dpa_res->start;
256104ad63f0SDan Williams m->size = resource_size(cxled->dpa_res);
256204ad63f0SDan Williams m->position = i;
256304ad63f0SDan Williams }
256404ad63f0SDan Williams
256504ad63f0SDan Williams dev = &cxlr_pmem->dev;
256604ad63f0SDan Williams cxlr_pmem->cxlr = cxlr;
2567f17b558dSDan Williams cxlr->cxlr_pmem = cxlr_pmem;
256804ad63f0SDan Williams device_initialize(dev);
256904ad63f0SDan Williams lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
257004ad63f0SDan Williams device_set_pm_not_required(dev);
257104ad63f0SDan Williams dev->parent = &cxlr->dev;
257204ad63f0SDan Williams dev->bus = &cxl_bus_type;
257304ad63f0SDan Williams dev->type = &cxl_pmem_region_type;
257404ad63f0SDan Williams out:
257504ad63f0SDan Williams up_read(&cxl_region_rwsem);
257604ad63f0SDan Williams
257704ad63f0SDan Williams return cxlr_pmem;
257804ad63f0SDan Williams }
257904ad63f0SDan Williams
cxl_dax_region_release(struct device * dev)258009d09e04SDan Williams static void cxl_dax_region_release(struct device *dev)
258109d09e04SDan Williams {
258209d09e04SDan Williams struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
258309d09e04SDan Williams
258409d09e04SDan Williams kfree(cxlr_dax);
258509d09e04SDan Williams }
258609d09e04SDan Williams
258709d09e04SDan Williams static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
258809d09e04SDan Williams &cxl_base_attribute_group,
258909d09e04SDan Williams NULL,
259009d09e04SDan Williams };
259109d09e04SDan Williams
259209d09e04SDan Williams const struct device_type cxl_dax_region_type = {
259309d09e04SDan Williams .name = "cxl_dax_region",
259409d09e04SDan Williams .release = cxl_dax_region_release,
259509d09e04SDan Williams .groups = cxl_dax_region_attribute_groups,
259609d09e04SDan Williams };
259709d09e04SDan Williams
is_cxl_dax_region(struct device * dev)259809d09e04SDan Williams static bool is_cxl_dax_region(struct device *dev)
259909d09e04SDan Williams {
260009d09e04SDan Williams return dev->type == &cxl_dax_region_type;
260109d09e04SDan Williams }
260209d09e04SDan Williams
to_cxl_dax_region(struct device * dev)260309d09e04SDan Williams struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
260409d09e04SDan Williams {
260509d09e04SDan Williams if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
260609d09e04SDan Williams "not a cxl_dax_region device\n"))
260709d09e04SDan Williams return NULL;
260809d09e04SDan Williams return container_of(dev, struct cxl_dax_region, dev);
260909d09e04SDan Williams }
261009d09e04SDan Williams EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
261109d09e04SDan Williams
261209d09e04SDan Williams static struct lock_class_key cxl_dax_region_key;
261309d09e04SDan Williams
cxl_dax_region_alloc(struct cxl_region * cxlr)261409d09e04SDan Williams static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
261509d09e04SDan Williams {
261609d09e04SDan Williams struct cxl_region_params *p = &cxlr->params;
261709d09e04SDan Williams struct cxl_dax_region *cxlr_dax;
261809d09e04SDan Williams struct device *dev;
261909d09e04SDan Williams
262009d09e04SDan Williams down_read(&cxl_region_rwsem);
262109d09e04SDan Williams if (p->state != CXL_CONFIG_COMMIT) {
262209d09e04SDan Williams cxlr_dax = ERR_PTR(-ENXIO);
262309d09e04SDan Williams goto out;
262409d09e04SDan Williams }
262509d09e04SDan Williams
262609d09e04SDan Williams cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
262709d09e04SDan Williams if (!cxlr_dax) {
262809d09e04SDan Williams cxlr_dax = ERR_PTR(-ENOMEM);
262909d09e04SDan Williams goto out;
263009d09e04SDan Williams }
263109d09e04SDan Williams
263209d09e04SDan Williams cxlr_dax->hpa_range.start = p->res->start;
263309d09e04SDan Williams cxlr_dax->hpa_range.end = p->res->end;
263409d09e04SDan Williams
263509d09e04SDan Williams dev = &cxlr_dax->dev;
263609d09e04SDan Williams cxlr_dax->cxlr = cxlr;
263709d09e04SDan Williams device_initialize(dev);
263809d09e04SDan Williams lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
263909d09e04SDan Williams device_set_pm_not_required(dev);
264009d09e04SDan Williams dev->parent = &cxlr->dev;
264109d09e04SDan Williams dev->bus = &cxl_bus_type;
264209d09e04SDan Williams dev->type = &cxl_dax_region_type;
264309d09e04SDan Williams out:
264409d09e04SDan Williams up_read(&cxl_region_rwsem);
264509d09e04SDan Williams
264609d09e04SDan Williams return cxlr_dax;
264709d09e04SDan Williams }
264809d09e04SDan Williams
cxlr_pmem_unregister(void * _cxlr_pmem)2649f17b558dSDan Williams static void cxlr_pmem_unregister(void *_cxlr_pmem)
265004ad63f0SDan Williams {
2651f17b558dSDan Williams struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2652f17b558dSDan Williams struct cxl_region *cxlr = cxlr_pmem->cxlr;
2653f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2654f17b558dSDan Williams
2655f17b558dSDan Williams /*
2656f17b558dSDan Williams * Either the bridge is in ->remove() context under the device_lock(),
2657f17b558dSDan Williams * or cxlr_release_nvdimm() is cancelling the bridge's release action
2658f17b558dSDan Williams * for @cxlr_pmem and doing it itself (while manually holding the bridge
2659f17b558dSDan Williams * lock).
2660f17b558dSDan Williams */
2661f17b558dSDan Williams device_lock_assert(&cxl_nvb->dev);
2662f17b558dSDan Williams cxlr->cxlr_pmem = NULL;
2663f17b558dSDan Williams cxlr_pmem->cxlr = NULL;
2664f17b558dSDan Williams device_unregister(&cxlr_pmem->dev);
2665f17b558dSDan Williams }
2666f17b558dSDan Williams
cxlr_release_nvdimm(void * _cxlr)2667f17b558dSDan Williams static void cxlr_release_nvdimm(void *_cxlr)
2668f17b558dSDan Williams {
2669f17b558dSDan Williams struct cxl_region *cxlr = _cxlr;
2670f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2671f17b558dSDan Williams
2672f17b558dSDan Williams device_lock(&cxl_nvb->dev);
2673f17b558dSDan Williams if (cxlr->cxlr_pmem)
2674f17b558dSDan Williams devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2675f17b558dSDan Williams cxlr->cxlr_pmem);
2676f17b558dSDan Williams device_unlock(&cxl_nvb->dev);
2677f17b558dSDan Williams cxlr->cxl_nvb = NULL;
2678f17b558dSDan Williams put_device(&cxl_nvb->dev);
267904ad63f0SDan Williams }
268004ad63f0SDan Williams
268104ad63f0SDan Williams /**
268204ad63f0SDan Williams * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
268304ad63f0SDan Williams * @cxlr: parent CXL region for this pmem region bridge device
268404ad63f0SDan Williams *
268504ad63f0SDan Williams * Return: 0 on success negative error code on failure.
268604ad63f0SDan Williams */
devm_cxl_add_pmem_region(struct cxl_region * cxlr)268704ad63f0SDan Williams static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
268804ad63f0SDan Williams {
268904ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem;
2690f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb;
269104ad63f0SDan Williams struct device *dev;
269204ad63f0SDan Williams int rc;
269304ad63f0SDan Williams
269404ad63f0SDan Williams cxlr_pmem = cxl_pmem_region_alloc(cxlr);
269504ad63f0SDan Williams if (IS_ERR(cxlr_pmem))
269604ad63f0SDan Williams return PTR_ERR(cxlr_pmem);
2697f17b558dSDan Williams cxl_nvb = cxlr->cxl_nvb;
269804ad63f0SDan Williams
269904ad63f0SDan Williams dev = &cxlr_pmem->dev;
270004ad63f0SDan Williams rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
270104ad63f0SDan Williams if (rc)
270204ad63f0SDan Williams goto err;
270304ad63f0SDan Williams
270404ad63f0SDan Williams rc = device_add(dev);
270504ad63f0SDan Williams if (rc)
270604ad63f0SDan Williams goto err;
270704ad63f0SDan Williams
270804ad63f0SDan Williams dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
270904ad63f0SDan Williams dev_name(dev));
271004ad63f0SDan Williams
2711f17b558dSDan Williams device_lock(&cxl_nvb->dev);
2712f17b558dSDan Williams if (cxl_nvb->dev.driver)
2713f17b558dSDan Williams rc = devm_add_action_or_reset(&cxl_nvb->dev,
2714f17b558dSDan Williams cxlr_pmem_unregister, cxlr_pmem);
2715f17b558dSDan Williams else
2716f17b558dSDan Williams rc = -ENXIO;
2717f17b558dSDan Williams device_unlock(&cxl_nvb->dev);
2718f17b558dSDan Williams
2719f17b558dSDan Williams if (rc)
2720f17b558dSDan Williams goto err_bridge;
2721f17b558dSDan Williams
2722f17b558dSDan Williams /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2723f17b558dSDan Williams return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
272404ad63f0SDan Williams
272504ad63f0SDan Williams err:
272604ad63f0SDan Williams put_device(dev);
2727f17b558dSDan Williams err_bridge:
2728f17b558dSDan Williams put_device(&cxl_nvb->dev);
2729f17b558dSDan Williams cxlr->cxl_nvb = NULL;
273004ad63f0SDan Williams return rc;
273104ad63f0SDan Williams }
273204ad63f0SDan Williams
cxlr_dax_unregister(void * _cxlr_dax)273309d09e04SDan Williams static void cxlr_dax_unregister(void *_cxlr_dax)
273409d09e04SDan Williams {
273509d09e04SDan Williams struct cxl_dax_region *cxlr_dax = _cxlr_dax;
273609d09e04SDan Williams
273709d09e04SDan Williams device_unregister(&cxlr_dax->dev);
273809d09e04SDan Williams }
273909d09e04SDan Williams
devm_cxl_add_dax_region(struct cxl_region * cxlr)274009d09e04SDan Williams static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
274109d09e04SDan Williams {
274209d09e04SDan Williams struct cxl_dax_region *cxlr_dax;
274309d09e04SDan Williams struct device *dev;
274409d09e04SDan Williams int rc;
274509d09e04SDan Williams
274609d09e04SDan Williams cxlr_dax = cxl_dax_region_alloc(cxlr);
274709d09e04SDan Williams if (IS_ERR(cxlr_dax))
274809d09e04SDan Williams return PTR_ERR(cxlr_dax);
274909d09e04SDan Williams
275009d09e04SDan Williams dev = &cxlr_dax->dev;
275109d09e04SDan Williams rc = dev_set_name(dev, "dax_region%d", cxlr->id);
275209d09e04SDan Williams if (rc)
275309d09e04SDan Williams goto err;
275409d09e04SDan Williams
275509d09e04SDan Williams rc = device_add(dev);
275609d09e04SDan Williams if (rc)
275709d09e04SDan Williams goto err;
275809d09e04SDan Williams
275909d09e04SDan Williams dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
276009d09e04SDan Williams dev_name(dev));
276109d09e04SDan Williams
276209d09e04SDan Williams return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
276309d09e04SDan Williams cxlr_dax);
276409d09e04SDan Williams err:
276509d09e04SDan Williams put_device(dev);
276609d09e04SDan Williams return rc;
276709d09e04SDan Williams }
276809d09e04SDan Williams
match_root_decoder_by_range(struct device * dev,void * data)2769c4255b9bSAlison Schofield static int match_root_decoder_by_range(struct device *dev, void *data)
2770a32320b7SDan Williams {
2771a32320b7SDan Williams struct range *r1, *r2 = data;
2772a32320b7SDan Williams struct cxl_root_decoder *cxlrd;
2773a32320b7SDan Williams
2774a32320b7SDan Williams if (!is_root_decoder(dev))
2775a32320b7SDan Williams return 0;
2776a32320b7SDan Williams
2777a32320b7SDan Williams cxlrd = to_cxl_root_decoder(dev);
2778a32320b7SDan Williams r1 = &cxlrd->cxlsd.cxld.hpa_range;
2779a32320b7SDan Williams return range_contains(r1, r2);
2780a32320b7SDan Williams }
2781a32320b7SDan Williams
match_region_by_range(struct device * dev,void * data)2782a32320b7SDan Williams static int match_region_by_range(struct device *dev, void *data)
2783a32320b7SDan Williams {
2784a32320b7SDan Williams struct cxl_region_params *p;
2785a32320b7SDan Williams struct cxl_region *cxlr;
2786a32320b7SDan Williams struct range *r = data;
2787a32320b7SDan Williams int rc = 0;
2788a32320b7SDan Williams
2789a32320b7SDan Williams if (!is_cxl_region(dev))
2790a32320b7SDan Williams return 0;
2791a32320b7SDan Williams
2792a32320b7SDan Williams cxlr = to_cxl_region(dev);
2793a32320b7SDan Williams p = &cxlr->params;
2794a32320b7SDan Williams
2795a32320b7SDan Williams down_read(&cxl_region_rwsem);
2796a32320b7SDan Williams if (p->res && p->res->start == r->start && p->res->end == r->end)
2797a32320b7SDan Williams rc = 1;
2798a32320b7SDan Williams up_read(&cxl_region_rwsem);
2799a32320b7SDan Williams
2800a32320b7SDan Williams return rc;
2801a32320b7SDan Williams }
2802a32320b7SDan Williams
2803a32320b7SDan Williams /* Establish an empty region covering the given HPA range */
construct_region(struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled)2804a32320b7SDan Williams static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2805a32320b7SDan Williams struct cxl_endpoint_decoder *cxled)
2806a32320b7SDan Williams {
2807a32320b7SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2808a32320b7SDan Williams struct cxl_port *port = cxlrd_to_port(cxlrd);
2809a32320b7SDan Williams struct range *hpa = &cxled->cxld.hpa_range;
2810a32320b7SDan Williams struct cxl_region_params *p;
2811a32320b7SDan Williams struct cxl_region *cxlr;
2812a32320b7SDan Williams struct resource *res;
2813a32320b7SDan Williams int rc;
2814a32320b7SDan Williams
2815a32320b7SDan Williams do {
2816a32320b7SDan Williams cxlr = __create_region(cxlrd, cxled->mode,
2817a32320b7SDan Williams atomic_read(&cxlrd->region_id));
2818a32320b7SDan Williams } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2819a32320b7SDan Williams
2820a32320b7SDan Williams if (IS_ERR(cxlr)) {
2821a32320b7SDan Williams dev_err(cxlmd->dev.parent,
2822a32320b7SDan Williams "%s:%s: %s failed assign region: %ld\n",
2823a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2824a32320b7SDan Williams __func__, PTR_ERR(cxlr));
2825a32320b7SDan Williams return cxlr;
2826a32320b7SDan Williams }
2827a32320b7SDan Williams
2828a32320b7SDan Williams down_write(&cxl_region_rwsem);
2829a32320b7SDan Williams p = &cxlr->params;
2830a32320b7SDan Williams if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2831a32320b7SDan Williams dev_err(cxlmd->dev.parent,
2832a32320b7SDan Williams "%s:%s: %s autodiscovery interrupted\n",
2833a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2834a32320b7SDan Williams __func__);
2835a32320b7SDan Williams rc = -EBUSY;
2836a32320b7SDan Williams goto err;
2837a32320b7SDan Williams }
2838a32320b7SDan Williams
2839a32320b7SDan Williams set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2840a32320b7SDan Williams
2841a32320b7SDan Williams res = kmalloc(sizeof(*res), GFP_KERNEL);
2842a32320b7SDan Williams if (!res) {
2843a32320b7SDan Williams rc = -ENOMEM;
2844a32320b7SDan Williams goto err;
2845a32320b7SDan Williams }
2846a32320b7SDan Williams
2847a32320b7SDan Williams *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2848a32320b7SDan Williams dev_name(&cxlr->dev));
2849a32320b7SDan Williams rc = insert_resource(cxlrd->res, res);
2850a32320b7SDan Williams if (rc) {
2851a32320b7SDan Williams /*
2852a32320b7SDan Williams * Platform-firmware may not have split resources like "System
2853a32320b7SDan Williams * RAM" on CXL window boundaries see cxl_region_iomem_release()
2854a32320b7SDan Williams */
2855a32320b7SDan Williams dev_warn(cxlmd->dev.parent,
2856a32320b7SDan Williams "%s:%s: %s %s cannot insert resource\n",
2857a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2858a32320b7SDan Williams __func__, dev_name(&cxlr->dev));
2859a32320b7SDan Williams }
2860a32320b7SDan Williams
2861a32320b7SDan Williams p->res = res;
2862a32320b7SDan Williams p->interleave_ways = cxled->cxld.interleave_ways;
2863a32320b7SDan Williams p->interleave_granularity = cxled->cxld.interleave_granularity;
2864a32320b7SDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2865a32320b7SDan Williams
2866a32320b7SDan Williams rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2867a32320b7SDan Williams if (rc)
2868a32320b7SDan Williams goto err;
2869a32320b7SDan Williams
2870a32320b7SDan Williams dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
2871a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
2872a32320b7SDan Williams dev_name(&cxlr->dev), p->res, p->interleave_ways,
2873a32320b7SDan Williams p->interleave_granularity);
2874a32320b7SDan Williams
2875a32320b7SDan Williams /* ...to match put_device() in cxl_add_to_region() */
2876a32320b7SDan Williams get_device(&cxlr->dev);
2877a32320b7SDan Williams up_write(&cxl_region_rwsem);
2878a32320b7SDan Williams
2879a32320b7SDan Williams return cxlr;
2880a32320b7SDan Williams
2881a32320b7SDan Williams err:
2882a32320b7SDan Williams up_write(&cxl_region_rwsem);
28837481653dSDan Williams devm_release_action(port->uport_dev, unregister_region, cxlr);
2884a32320b7SDan Williams return ERR_PTR(rc);
2885a32320b7SDan Williams }
2886a32320b7SDan Williams
cxl_add_to_region(struct cxl_port * root,struct cxl_endpoint_decoder * cxled)2887a32320b7SDan Williams int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
2888a32320b7SDan Williams {
2889a32320b7SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2890a32320b7SDan Williams struct range *hpa = &cxled->cxld.hpa_range;
2891a32320b7SDan Williams struct cxl_decoder *cxld = &cxled->cxld;
2892a32320b7SDan Williams struct device *cxlrd_dev, *region_dev;
2893a32320b7SDan Williams struct cxl_root_decoder *cxlrd;
2894a32320b7SDan Williams struct cxl_region_params *p;
2895a32320b7SDan Williams struct cxl_region *cxlr;
2896a32320b7SDan Williams bool attach = false;
2897a32320b7SDan Williams int rc;
2898a32320b7SDan Williams
2899a32320b7SDan Williams cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
2900c4255b9bSAlison Schofield match_root_decoder_by_range);
2901a32320b7SDan Williams if (!cxlrd_dev) {
2902a32320b7SDan Williams dev_err(cxlmd->dev.parent,
2903a32320b7SDan Williams "%s:%s no CXL window for range %#llx:%#llx\n",
2904a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxld->dev),
2905a32320b7SDan Williams cxld->hpa_range.start, cxld->hpa_range.end);
2906a32320b7SDan Williams return -ENXIO;
2907a32320b7SDan Williams }
2908a32320b7SDan Williams
2909a32320b7SDan Williams cxlrd = to_cxl_root_decoder(cxlrd_dev);
2910a32320b7SDan Williams
2911a32320b7SDan Williams /*
2912a32320b7SDan Williams * Ensure that if multiple threads race to construct_region() for @hpa
2913a32320b7SDan Williams * one does the construction and the others add to that.
2914a32320b7SDan Williams */
2915a32320b7SDan Williams mutex_lock(&cxlrd->range_lock);
2916a32320b7SDan Williams region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
2917a32320b7SDan Williams match_region_by_range);
2918a32320b7SDan Williams if (!region_dev) {
2919a32320b7SDan Williams cxlr = construct_region(cxlrd, cxled);
2920a32320b7SDan Williams region_dev = &cxlr->dev;
2921a32320b7SDan Williams } else
2922a32320b7SDan Williams cxlr = to_cxl_region(region_dev);
2923a32320b7SDan Williams mutex_unlock(&cxlrd->range_lock);
2924a32320b7SDan Williams
29257abcb0b1SArnd Bergmann rc = PTR_ERR_OR_ZERO(cxlr);
29267abcb0b1SArnd Bergmann if (rc)
2927a32320b7SDan Williams goto out;
2928a32320b7SDan Williams
2929a32320b7SDan Williams attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
2930a32320b7SDan Williams
2931a32320b7SDan Williams down_read(&cxl_region_rwsem);
2932a32320b7SDan Williams p = &cxlr->params;
2933a32320b7SDan Williams attach = p->state == CXL_CONFIG_COMMIT;
2934a32320b7SDan Williams up_read(&cxl_region_rwsem);
2935a32320b7SDan Williams
2936a32320b7SDan Williams if (attach) {
2937a32320b7SDan Williams /*
2938a32320b7SDan Williams * If device_attach() fails the range may still be active via
2939a32320b7SDan Williams * the platform-firmware memory map, otherwise the driver for
2940a32320b7SDan Williams * regions is local to this file, so driver matching can't fail.
2941a32320b7SDan Williams */
2942a32320b7SDan Williams if (device_attach(&cxlr->dev) < 0)
2943a32320b7SDan Williams dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
2944a32320b7SDan Williams p->res);
2945a32320b7SDan Williams }
2946a32320b7SDan Williams
2947a32320b7SDan Williams put_device(region_dev);
2948a32320b7SDan Williams out:
2949a32320b7SDan Williams put_device(cxlrd_dev);
2950a32320b7SDan Williams return rc;
2951a32320b7SDan Williams }
2952a32320b7SDan Williams EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
2953a32320b7SDan Williams
is_system_ram(struct resource * res,void * arg)2954a32320b7SDan Williams static int is_system_ram(struct resource *res, void *arg)
2955a32320b7SDan Williams {
2956a32320b7SDan Williams struct cxl_region *cxlr = arg;
2957a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params;
2958a32320b7SDan Williams
2959a32320b7SDan Williams dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
2960a32320b7SDan Williams return 1;
2961a32320b7SDan Williams }
2962a32320b7SDan Williams
cxl_region_probe(struct device * dev)29638d48817dSDan Williams static int cxl_region_probe(struct device *dev)
29648d48817dSDan Williams {
29658d48817dSDan Williams struct cxl_region *cxlr = to_cxl_region(dev);
29668d48817dSDan Williams struct cxl_region_params *p = &cxlr->params;
29678d48817dSDan Williams int rc;
29688d48817dSDan Williams
29698d48817dSDan Williams rc = down_read_interruptible(&cxl_region_rwsem);
29708d48817dSDan Williams if (rc) {
29718d48817dSDan Williams dev_dbg(&cxlr->dev, "probe interrupted\n");
29728d48817dSDan Williams return rc;
29738d48817dSDan Williams }
29748d48817dSDan Williams
29758d48817dSDan Williams if (p->state < CXL_CONFIG_COMMIT) {
29768d48817dSDan Williams dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
29778d48817dSDan Williams rc = -ENXIO;
2978d18bc74aSDan Williams goto out;
29798d48817dSDan Williams }
29808d48817dSDan Williams
29812ab47045SDan Williams if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
29822ab47045SDan Williams dev_err(&cxlr->dev,
29832ab47045SDan Williams "failed to activate, re-commit region and retry\n");
29842ab47045SDan Williams rc = -ENXIO;
29852ab47045SDan Williams goto out;
29862ab47045SDan Williams }
2987d18bc74aSDan Williams
29888d48817dSDan Williams /*
29898d48817dSDan Williams * From this point on any path that changes the region's state away from
29908d48817dSDan Williams * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
29918d48817dSDan Williams */
2992d18bc74aSDan Williams out:
29938d48817dSDan Williams up_read(&cxl_region_rwsem);
29948d48817dSDan Williams
2995bf3e5da8SDan Williams if (rc)
2996bf3e5da8SDan Williams return rc;
2997bf3e5da8SDan Williams
299804ad63f0SDan Williams switch (cxlr->mode) {
299904ad63f0SDan Williams case CXL_DECODER_PMEM:
300004ad63f0SDan Williams return devm_cxl_add_pmem_region(cxlr);
3001a32320b7SDan Williams case CXL_DECODER_RAM:
3002a32320b7SDan Williams /*
3003a32320b7SDan Williams * The region can not be manged by CXL if any portion of
3004a32320b7SDan Williams * it is already online as 'System RAM'
3005a32320b7SDan Williams */
3006a32320b7SDan Williams if (walk_iomem_res_desc(IORES_DESC_NONE,
3007a32320b7SDan Williams IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3008a32320b7SDan Williams p->res->start, p->res->end, cxlr,
3009a32320b7SDan Williams is_system_ram) > 0)
3010a32320b7SDan Williams return 0;
301109d09e04SDan Williams return devm_cxl_add_dax_region(cxlr);
301204ad63f0SDan Williams default:
301304ad63f0SDan Williams dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
301404ad63f0SDan Williams cxlr->mode);
301504ad63f0SDan Williams return -ENXIO;
301604ad63f0SDan Williams }
30178d48817dSDan Williams }
30188d48817dSDan Williams
30198d48817dSDan Williams static struct cxl_driver cxl_region_driver = {
30208d48817dSDan Williams .name = "cxl_region",
30218d48817dSDan Williams .probe = cxl_region_probe,
30228d48817dSDan Williams .id = CXL_DEVICE_REGION,
30238d48817dSDan Williams };
30248d48817dSDan Williams
cxl_region_init(void)30258d48817dSDan Williams int cxl_region_init(void)
30268d48817dSDan Williams {
30278d48817dSDan Williams return cxl_driver_register(&cxl_region_driver);
30288d48817dSDan Williams }
30298d48817dSDan Williams
cxl_region_exit(void)30308d48817dSDan Williams void cxl_region_exit(void)
30318d48817dSDan Williams {
30328d48817dSDan Williams cxl_driver_unregister(&cxl_region_driver);
30338d48817dSDan Williams }
30348d48817dSDan Williams
303523a22cd1SDan Williams MODULE_IMPORT_NS(CXL);
3036d18bc74aSDan Williams MODULE_IMPORT_NS(DEVMEM);
30378d48817dSDan Williams MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
3038