1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/memregion.h>
4 #include <linux/genalloc.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/slab.h>
8 #include <linux/uuid.h>
9 #include <linux/sort.h>
10 #include <linux/idr.h>
11 #include <cxlmem.h>
12 #include <cxl.h>
13 #include "core.h"
14
15 /**
16 * DOC: cxl core region
17 *
18 * CXL Regions represent mapped memory capacity in system physical address
19 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
20 * Memory ranges, Regions represent the active mapped capacity by the HDM
21 * Decoder Capability structures throughout the Host Bridges, Switches, and
22 * Endpoints in the topology.
23 *
24 * Region configuration has ordering constraints. UUID may be set at any time
25 * but is only visible for persistent regions.
26 * 1. Interleave granularity
27 * 2. Interleave size
28 * 3. Decoder targets
29 */
30
31 static struct cxl_region *to_cxl_region(struct device *dev);
32
uuid_show(struct device * dev,struct device_attribute * attr,char * buf)33 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
34 char *buf)
35 {
36 struct cxl_region *cxlr = to_cxl_region(dev);
37 struct cxl_region_params *p = &cxlr->params;
38 ssize_t rc;
39
40 rc = down_read_interruptible(&cxl_region_rwsem);
41 if (rc)
42 return rc;
43 if (cxlr->mode != CXL_DECODER_PMEM)
44 rc = sysfs_emit(buf, "\n");
45 else
46 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
47 up_read(&cxl_region_rwsem);
48
49 return rc;
50 }
51
is_dup(struct device * match,void * data)52 static int is_dup(struct device *match, void *data)
53 {
54 struct cxl_region_params *p;
55 struct cxl_region *cxlr;
56 uuid_t *uuid = data;
57
58 if (!is_cxl_region(match))
59 return 0;
60
61 lockdep_assert_held(&cxl_region_rwsem);
62 cxlr = to_cxl_region(match);
63 p = &cxlr->params;
64
65 if (uuid_equal(&p->uuid, uuid)) {
66 dev_dbg(match, "already has uuid: %pUb\n", uuid);
67 return -EBUSY;
68 }
69
70 return 0;
71 }
72
uuid_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)73 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
74 const char *buf, size_t len)
75 {
76 struct cxl_region *cxlr = to_cxl_region(dev);
77 struct cxl_region_params *p = &cxlr->params;
78 uuid_t temp;
79 ssize_t rc;
80
81 if (len != UUID_STRING_LEN + 1)
82 return -EINVAL;
83
84 rc = uuid_parse(buf, &temp);
85 if (rc)
86 return rc;
87
88 if (uuid_is_null(&temp))
89 return -EINVAL;
90
91 rc = down_write_killable(&cxl_region_rwsem);
92 if (rc)
93 return rc;
94
95 if (uuid_equal(&p->uuid, &temp))
96 goto out;
97
98 rc = -EBUSY;
99 if (p->state >= CXL_CONFIG_ACTIVE)
100 goto out;
101
102 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
103 if (rc < 0)
104 goto out;
105
106 uuid_copy(&p->uuid, &temp);
107 out:
108 up_write(&cxl_region_rwsem);
109
110 if (rc)
111 return rc;
112 return len;
113 }
114 static DEVICE_ATTR_RW(uuid);
115
cxl_rr_load(struct cxl_port * port,struct cxl_region * cxlr)116 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
117 struct cxl_region *cxlr)
118 {
119 return xa_load(&port->regions, (unsigned long)cxlr);
120 }
121
cxl_region_invalidate_memregion(struct cxl_region * cxlr)122 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
123 {
124 if (!cpu_cache_has_invalidate_memregion()) {
125 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
126 dev_warn_once(
127 &cxlr->dev,
128 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
129 return 0;
130 } else {
131 dev_WARN(&cxlr->dev,
132 "Failed to synchronize CPU cache state\n");
133 return -ENXIO;
134 }
135 }
136
137 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
138 return 0;
139 }
140
cxl_region_decode_reset(struct cxl_region * cxlr,int count)141 static void cxl_region_decode_reset(struct cxl_region *cxlr, int count)
142 {
143 struct cxl_region_params *p = &cxlr->params;
144 int i;
145
146 /*
147 * Before region teardown attempt to flush, evict any data cached for
148 * this region, or scream loudly about missing arch / platform support
149 * for CXL teardown.
150 */
151 cxl_region_invalidate_memregion(cxlr);
152
153 for (i = count - 1; i >= 0; i--) {
154 struct cxl_endpoint_decoder *cxled = p->targets[i];
155 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
156 struct cxl_port *iter = cxled_to_port(cxled);
157 struct cxl_dev_state *cxlds = cxlmd->cxlds;
158 struct cxl_ep *ep;
159
160 if (cxlds->rcd)
161 goto endpoint_reset;
162
163 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
164 iter = to_cxl_port(iter->dev.parent);
165
166 for (ep = cxl_ep_load(iter, cxlmd); iter;
167 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
168 struct cxl_region_ref *cxl_rr;
169 struct cxl_decoder *cxld;
170
171 cxl_rr = cxl_rr_load(iter, cxlr);
172 cxld = cxl_rr->decoder;
173 if (cxld->reset)
174 cxld->reset(cxld);
175 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
176 }
177
178 endpoint_reset:
179 cxled->cxld.reset(&cxled->cxld);
180 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
181 }
182
183 /* all decoders associated with this region have been torn down */
184 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
185 }
186
commit_decoder(struct cxl_decoder * cxld)187 static int commit_decoder(struct cxl_decoder *cxld)
188 {
189 struct cxl_switch_decoder *cxlsd = NULL;
190
191 if (cxld->commit)
192 return cxld->commit(cxld);
193
194 if (is_switch_decoder(&cxld->dev))
195 cxlsd = to_cxl_switch_decoder(&cxld->dev);
196
197 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
198 "->commit() is required\n"))
199 return -ENXIO;
200 return 0;
201 }
202
cxl_region_decode_commit(struct cxl_region * cxlr)203 static int cxl_region_decode_commit(struct cxl_region *cxlr)
204 {
205 struct cxl_region_params *p = &cxlr->params;
206 int i, rc = 0;
207
208 for (i = 0; i < p->nr_targets; i++) {
209 struct cxl_endpoint_decoder *cxled = p->targets[i];
210 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
211 struct cxl_region_ref *cxl_rr;
212 struct cxl_decoder *cxld;
213 struct cxl_port *iter;
214 struct cxl_ep *ep;
215
216 /* commit bottom up */
217 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
218 iter = to_cxl_port(iter->dev.parent)) {
219 cxl_rr = cxl_rr_load(iter, cxlr);
220 cxld = cxl_rr->decoder;
221 rc = commit_decoder(cxld);
222 if (rc)
223 break;
224 }
225
226 if (rc) {
227 /* programming @iter failed, teardown */
228 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
229 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
230 cxl_rr = cxl_rr_load(iter, cxlr);
231 cxld = cxl_rr->decoder;
232 if (cxld->reset)
233 cxld->reset(cxld);
234 }
235
236 cxled->cxld.reset(&cxled->cxld);
237 goto err;
238 }
239 }
240
241 return 0;
242
243 err:
244 /* undo the targets that were successfully committed */
245 cxl_region_decode_reset(cxlr, i);
246 return rc;
247 }
248
commit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)249 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
250 const char *buf, size_t len)
251 {
252 struct cxl_region *cxlr = to_cxl_region(dev);
253 struct cxl_region_params *p = &cxlr->params;
254 bool commit;
255 ssize_t rc;
256
257 rc = kstrtobool(buf, &commit);
258 if (rc)
259 return rc;
260
261 rc = down_write_killable(&cxl_region_rwsem);
262 if (rc)
263 return rc;
264
265 /* Already in the requested state? */
266 if (commit && p->state >= CXL_CONFIG_COMMIT)
267 goto out;
268 if (!commit && p->state < CXL_CONFIG_COMMIT)
269 goto out;
270
271 /* Not ready to commit? */
272 if (commit && p->state < CXL_CONFIG_ACTIVE) {
273 rc = -ENXIO;
274 goto out;
275 }
276
277 /*
278 * Invalidate caches before region setup to drop any speculative
279 * consumption of this address space
280 */
281 rc = cxl_region_invalidate_memregion(cxlr);
282 if (rc)
283 goto out;
284
285 if (commit) {
286 rc = cxl_region_decode_commit(cxlr);
287 if (rc == 0)
288 p->state = CXL_CONFIG_COMMIT;
289 } else {
290 p->state = CXL_CONFIG_RESET_PENDING;
291 up_write(&cxl_region_rwsem);
292 device_release_driver(&cxlr->dev);
293 down_write(&cxl_region_rwsem);
294
295 /*
296 * The lock was dropped, so need to revalidate that the reset is
297 * still pending.
298 */
299 if (p->state == CXL_CONFIG_RESET_PENDING) {
300 cxl_region_decode_reset(cxlr, p->interleave_ways);
301 p->state = CXL_CONFIG_ACTIVE;
302 }
303 }
304
305 out:
306 up_write(&cxl_region_rwsem);
307
308 if (rc)
309 return rc;
310 return len;
311 }
312
commit_show(struct device * dev,struct device_attribute * attr,char * buf)313 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
314 char *buf)
315 {
316 struct cxl_region *cxlr = to_cxl_region(dev);
317 struct cxl_region_params *p = &cxlr->params;
318 ssize_t rc;
319
320 rc = down_read_interruptible(&cxl_region_rwsem);
321 if (rc)
322 return rc;
323 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
324 up_read(&cxl_region_rwsem);
325
326 return rc;
327 }
328 static DEVICE_ATTR_RW(commit);
329
cxl_region_visible(struct kobject * kobj,struct attribute * a,int n)330 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
331 int n)
332 {
333 struct device *dev = kobj_to_dev(kobj);
334 struct cxl_region *cxlr = to_cxl_region(dev);
335
336 /*
337 * Support tooling that expects to find a 'uuid' attribute for all
338 * regions regardless of mode.
339 */
340 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
341 return 0444;
342 return a->mode;
343 }
344
interleave_ways_show(struct device * dev,struct device_attribute * attr,char * buf)345 static ssize_t interleave_ways_show(struct device *dev,
346 struct device_attribute *attr, char *buf)
347 {
348 struct cxl_region *cxlr = to_cxl_region(dev);
349 struct cxl_region_params *p = &cxlr->params;
350 ssize_t rc;
351
352 rc = down_read_interruptible(&cxl_region_rwsem);
353 if (rc)
354 return rc;
355 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
356 up_read(&cxl_region_rwsem);
357
358 return rc;
359 }
360
361 static const struct attribute_group *get_cxl_region_target_group(void);
362
interleave_ways_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)363 static ssize_t interleave_ways_store(struct device *dev,
364 struct device_attribute *attr,
365 const char *buf, size_t len)
366 {
367 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
368 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
369 struct cxl_region *cxlr = to_cxl_region(dev);
370 struct cxl_region_params *p = &cxlr->params;
371 unsigned int val, save;
372 int rc;
373 u8 iw;
374
375 rc = kstrtouint(buf, 0, &val);
376 if (rc)
377 return rc;
378
379 rc = ways_to_eiw(val, &iw);
380 if (rc)
381 return rc;
382
383 /*
384 * Even for x3, x6, and x12 interleaves the region interleave must be a
385 * power of 2 multiple of the host bridge interleave.
386 */
387 if (!is_power_of_2(val / cxld->interleave_ways) ||
388 (val % cxld->interleave_ways)) {
389 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
390 return -EINVAL;
391 }
392
393 rc = down_write_killable(&cxl_region_rwsem);
394 if (rc)
395 return rc;
396 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
397 rc = -EBUSY;
398 goto out;
399 }
400
401 save = p->interleave_ways;
402 p->interleave_ways = val;
403 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
404 if (rc)
405 p->interleave_ways = save;
406 out:
407 up_write(&cxl_region_rwsem);
408 if (rc)
409 return rc;
410 return len;
411 }
412 static DEVICE_ATTR_RW(interleave_ways);
413
interleave_granularity_show(struct device * dev,struct device_attribute * attr,char * buf)414 static ssize_t interleave_granularity_show(struct device *dev,
415 struct device_attribute *attr,
416 char *buf)
417 {
418 struct cxl_region *cxlr = to_cxl_region(dev);
419 struct cxl_region_params *p = &cxlr->params;
420 ssize_t rc;
421
422 rc = down_read_interruptible(&cxl_region_rwsem);
423 if (rc)
424 return rc;
425 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
426 up_read(&cxl_region_rwsem);
427
428 return rc;
429 }
430
interleave_granularity_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)431 static ssize_t interleave_granularity_store(struct device *dev,
432 struct device_attribute *attr,
433 const char *buf, size_t len)
434 {
435 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
436 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
437 struct cxl_region *cxlr = to_cxl_region(dev);
438 struct cxl_region_params *p = &cxlr->params;
439 int rc, val;
440 u16 ig;
441
442 rc = kstrtoint(buf, 0, &val);
443 if (rc)
444 return rc;
445
446 rc = granularity_to_eig(val, &ig);
447 if (rc)
448 return rc;
449
450 /*
451 * When the host-bridge is interleaved, disallow region granularity !=
452 * root granularity. Regions with a granularity less than the root
453 * interleave result in needing multiple endpoints to support a single
454 * slot in the interleave (possible to support in the future). Regions
455 * with a granularity greater than the root interleave result in invalid
456 * DPA translations (invalid to support).
457 */
458 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
459 return -EINVAL;
460
461 rc = down_write_killable(&cxl_region_rwsem);
462 if (rc)
463 return rc;
464 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
465 rc = -EBUSY;
466 goto out;
467 }
468
469 p->interleave_granularity = val;
470 out:
471 up_write(&cxl_region_rwsem);
472 if (rc)
473 return rc;
474 return len;
475 }
476 static DEVICE_ATTR_RW(interleave_granularity);
477
resource_show(struct device * dev,struct device_attribute * attr,char * buf)478 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
479 char *buf)
480 {
481 struct cxl_region *cxlr = to_cxl_region(dev);
482 struct cxl_region_params *p = &cxlr->params;
483 u64 resource = -1ULL;
484 ssize_t rc;
485
486 rc = down_read_interruptible(&cxl_region_rwsem);
487 if (rc)
488 return rc;
489 if (p->res)
490 resource = p->res->start;
491 rc = sysfs_emit(buf, "%#llx\n", resource);
492 up_read(&cxl_region_rwsem);
493
494 return rc;
495 }
496 static DEVICE_ATTR_RO(resource);
497
mode_show(struct device * dev,struct device_attribute * attr,char * buf)498 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
499 char *buf)
500 {
501 struct cxl_region *cxlr = to_cxl_region(dev);
502
503 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
504 }
505 static DEVICE_ATTR_RO(mode);
506
alloc_hpa(struct cxl_region * cxlr,resource_size_t size)507 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
508 {
509 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
510 struct cxl_region_params *p = &cxlr->params;
511 struct resource *res;
512 u64 remainder = 0;
513
514 lockdep_assert_held_write(&cxl_region_rwsem);
515
516 /* Nothing to do... */
517 if (p->res && resource_size(p->res) == size)
518 return 0;
519
520 /* To change size the old size must be freed first */
521 if (p->res)
522 return -EBUSY;
523
524 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
525 return -EBUSY;
526
527 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
528 if (!p->interleave_ways || !p->interleave_granularity ||
529 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
530 return -ENXIO;
531
532 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
533 if (remainder)
534 return -EINVAL;
535
536 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
537 dev_name(&cxlr->dev));
538 if (IS_ERR(res)) {
539 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
540 PTR_ERR(res));
541 return PTR_ERR(res);
542 }
543
544 p->res = res;
545 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
546
547 return 0;
548 }
549
cxl_region_iomem_release(struct cxl_region * cxlr)550 static void cxl_region_iomem_release(struct cxl_region *cxlr)
551 {
552 struct cxl_region_params *p = &cxlr->params;
553
554 if (device_is_registered(&cxlr->dev))
555 lockdep_assert_held_write(&cxl_region_rwsem);
556 if (p->res) {
557 /*
558 * Autodiscovered regions may not have been able to insert their
559 * resource.
560 */
561 if (p->res->parent)
562 remove_resource(p->res);
563 kfree(p->res);
564 p->res = NULL;
565 }
566 }
567
free_hpa(struct cxl_region * cxlr)568 static int free_hpa(struct cxl_region *cxlr)
569 {
570 struct cxl_region_params *p = &cxlr->params;
571
572 lockdep_assert_held_write(&cxl_region_rwsem);
573
574 if (!p->res)
575 return 0;
576
577 if (p->state >= CXL_CONFIG_ACTIVE)
578 return -EBUSY;
579
580 cxl_region_iomem_release(cxlr);
581 p->state = CXL_CONFIG_IDLE;
582 return 0;
583 }
584
size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)585 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
586 const char *buf, size_t len)
587 {
588 struct cxl_region *cxlr = to_cxl_region(dev);
589 u64 val;
590 int rc;
591
592 rc = kstrtou64(buf, 0, &val);
593 if (rc)
594 return rc;
595
596 rc = down_write_killable(&cxl_region_rwsem);
597 if (rc)
598 return rc;
599
600 if (val)
601 rc = alloc_hpa(cxlr, val);
602 else
603 rc = free_hpa(cxlr);
604 up_write(&cxl_region_rwsem);
605
606 if (rc)
607 return rc;
608
609 return len;
610 }
611
size_show(struct device * dev,struct device_attribute * attr,char * buf)612 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
613 char *buf)
614 {
615 struct cxl_region *cxlr = to_cxl_region(dev);
616 struct cxl_region_params *p = &cxlr->params;
617 u64 size = 0;
618 ssize_t rc;
619
620 rc = down_read_interruptible(&cxl_region_rwsem);
621 if (rc)
622 return rc;
623 if (p->res)
624 size = resource_size(p->res);
625 rc = sysfs_emit(buf, "%#llx\n", size);
626 up_read(&cxl_region_rwsem);
627
628 return rc;
629 }
630 static DEVICE_ATTR_RW(size);
631
632 static struct attribute *cxl_region_attrs[] = {
633 &dev_attr_uuid.attr,
634 &dev_attr_commit.attr,
635 &dev_attr_interleave_ways.attr,
636 &dev_attr_interleave_granularity.attr,
637 &dev_attr_resource.attr,
638 &dev_attr_size.attr,
639 &dev_attr_mode.attr,
640 NULL,
641 };
642
643 static const struct attribute_group cxl_region_group = {
644 .attrs = cxl_region_attrs,
645 .is_visible = cxl_region_visible,
646 };
647
show_targetN(struct cxl_region * cxlr,char * buf,int pos)648 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
649 {
650 struct cxl_region_params *p = &cxlr->params;
651 struct cxl_endpoint_decoder *cxled;
652 int rc;
653
654 rc = down_read_interruptible(&cxl_region_rwsem);
655 if (rc)
656 return rc;
657
658 if (pos >= p->interleave_ways) {
659 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
660 p->interleave_ways);
661 rc = -ENXIO;
662 goto out;
663 }
664
665 cxled = p->targets[pos];
666 if (!cxled)
667 rc = sysfs_emit(buf, "\n");
668 else
669 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
670 out:
671 up_read(&cxl_region_rwsem);
672
673 return rc;
674 }
675
match_free_decoder(struct device * dev,void * data)676 static int match_free_decoder(struct device *dev, void *data)
677 {
678 struct cxl_decoder *cxld;
679 int *id = data;
680
681 if (!is_switch_decoder(dev))
682 return 0;
683
684 cxld = to_cxl_decoder(dev);
685
686 /* enforce ordered allocation */
687 if (cxld->id != *id)
688 return 0;
689
690 if (!cxld->region)
691 return 1;
692
693 (*id)++;
694
695 return 0;
696 }
697
match_auto_decoder(struct device * dev,void * data)698 static int match_auto_decoder(struct device *dev, void *data)
699 {
700 struct cxl_region_params *p = data;
701 struct cxl_decoder *cxld;
702 struct range *r;
703
704 if (!is_switch_decoder(dev))
705 return 0;
706
707 cxld = to_cxl_decoder(dev);
708 r = &cxld->hpa_range;
709
710 if (p->res && p->res->start == r->start && p->res->end == r->end)
711 return 1;
712
713 return 0;
714 }
715
716 static struct cxl_decoder *
cxl_region_find_decoder(struct cxl_port * port,struct cxl_endpoint_decoder * cxled,struct cxl_region * cxlr)717 cxl_region_find_decoder(struct cxl_port *port,
718 struct cxl_endpoint_decoder *cxled,
719 struct cxl_region *cxlr)
720 {
721 struct device *dev;
722 int id = 0;
723
724 if (port == cxled_to_port(cxled))
725 return &cxled->cxld;
726
727 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
728 dev = device_find_child(&port->dev, &cxlr->params,
729 match_auto_decoder);
730 else
731 dev = device_find_child(&port->dev, &id, match_free_decoder);
732 if (!dev)
733 return NULL;
734 /*
735 * This decoder is pinned registered as long as the endpoint decoder is
736 * registered, and endpoint decoder unregistration holds the
737 * cxl_region_rwsem over unregister events, so no need to hold on to
738 * this extra reference.
739 */
740 put_device(dev);
741 return to_cxl_decoder(dev);
742 }
743
auto_order_ok(struct cxl_port * port,struct cxl_region * cxlr_iter,struct cxl_decoder * cxld)744 static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
745 struct cxl_decoder *cxld)
746 {
747 struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
748 struct cxl_decoder *cxld_iter = rr->decoder;
749
750 /*
751 * Allow the out of order assembly of auto-discovered regions.
752 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
753 * in HPA order. Confirm that the decoder with the lesser HPA
754 * starting address has the lesser id.
755 */
756 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
757 dev_name(&cxld->dev), cxld->id,
758 dev_name(&cxld_iter->dev), cxld_iter->id);
759
760 if (cxld_iter->id > cxld->id)
761 return true;
762
763 return false;
764 }
765
766 static struct cxl_region_ref *
alloc_region_ref(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)767 alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
768 struct cxl_endpoint_decoder *cxled)
769 {
770 struct cxl_region_params *p = &cxlr->params;
771 struct cxl_region_ref *cxl_rr, *iter;
772 unsigned long index;
773 int rc;
774
775 xa_for_each(&port->regions, index, iter) {
776 struct cxl_region_params *ip = &iter->region->params;
777
778 if (!ip->res || ip->res->start < p->res->start)
779 continue;
780
781 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
782 struct cxl_decoder *cxld;
783
784 cxld = cxl_region_find_decoder(port, cxled, cxlr);
785 if (auto_order_ok(port, iter->region, cxld))
786 continue;
787 }
788 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
789 dev_name(&port->dev),
790 dev_name(&iter->region->dev), ip->res, p->res);
791
792 return ERR_PTR(-EBUSY);
793 }
794
795 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
796 if (!cxl_rr)
797 return ERR_PTR(-ENOMEM);
798 cxl_rr->port = port;
799 cxl_rr->region = cxlr;
800 cxl_rr->nr_targets = 1;
801 xa_init(&cxl_rr->endpoints);
802
803 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
804 if (rc) {
805 dev_dbg(&cxlr->dev,
806 "%s: failed to track region reference: %d\n",
807 dev_name(&port->dev), rc);
808 kfree(cxl_rr);
809 return ERR_PTR(rc);
810 }
811
812 return cxl_rr;
813 }
814
cxl_rr_free_decoder(struct cxl_region_ref * cxl_rr)815 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
816 {
817 struct cxl_region *cxlr = cxl_rr->region;
818 struct cxl_decoder *cxld = cxl_rr->decoder;
819
820 if (!cxld)
821 return;
822
823 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
824 if (cxld->region == cxlr) {
825 cxld->region = NULL;
826 put_device(&cxlr->dev);
827 }
828 }
829
free_region_ref(struct cxl_region_ref * cxl_rr)830 static void free_region_ref(struct cxl_region_ref *cxl_rr)
831 {
832 struct cxl_port *port = cxl_rr->port;
833 struct cxl_region *cxlr = cxl_rr->region;
834
835 cxl_rr_free_decoder(cxl_rr);
836 xa_erase(&port->regions, (unsigned long)cxlr);
837 xa_destroy(&cxl_rr->endpoints);
838 kfree(cxl_rr);
839 }
840
cxl_rr_ep_add(struct cxl_region_ref * cxl_rr,struct cxl_endpoint_decoder * cxled)841 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
842 struct cxl_endpoint_decoder *cxled)
843 {
844 int rc;
845 struct cxl_port *port = cxl_rr->port;
846 struct cxl_region *cxlr = cxl_rr->region;
847 struct cxl_decoder *cxld = cxl_rr->decoder;
848 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
849
850 if (ep) {
851 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
852 GFP_KERNEL);
853 if (rc)
854 return rc;
855 }
856 cxl_rr->nr_eps++;
857
858 if (!cxld->region) {
859 cxld->region = cxlr;
860 get_device(&cxlr->dev);
861 }
862
863 return 0;
864 }
865
cxl_rr_alloc_decoder(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,struct cxl_region_ref * cxl_rr)866 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
867 struct cxl_endpoint_decoder *cxled,
868 struct cxl_region_ref *cxl_rr)
869 {
870 struct cxl_decoder *cxld;
871
872 cxld = cxl_region_find_decoder(port, cxled, cxlr);
873 if (!cxld) {
874 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
875 dev_name(&port->dev));
876 return -EBUSY;
877 }
878
879 if (cxld->region) {
880 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
881 dev_name(&port->dev), dev_name(&cxld->dev),
882 dev_name(&cxld->region->dev));
883 return -EBUSY;
884 }
885
886 /*
887 * Endpoints should already match the region type, but backstop that
888 * assumption with an assertion. Switch-decoders change mapping-type
889 * based on what is mapped when they are assigned to a region.
890 */
891 dev_WARN_ONCE(&cxlr->dev,
892 port == cxled_to_port(cxled) &&
893 cxld->target_type != cxlr->type,
894 "%s:%s mismatch decoder type %d -> %d\n",
895 dev_name(&cxled_to_memdev(cxled)->dev),
896 dev_name(&cxld->dev), cxld->target_type, cxlr->type);
897 cxld->target_type = cxlr->type;
898 cxl_rr->decoder = cxld;
899 return 0;
900 }
901
902 /**
903 * cxl_port_attach_region() - track a region's interest in a port by endpoint
904 * @port: port to add a new region reference 'struct cxl_region_ref'
905 * @cxlr: region to attach to @port
906 * @cxled: endpoint decoder used to create or further pin a region reference
907 * @pos: interleave position of @cxled in @cxlr
908 *
909 * The attach event is an opportunity to validate CXL decode setup
910 * constraints and record metadata needed for programming HDM decoders,
911 * in particular decoder target lists.
912 *
913 * The steps are:
914 *
915 * - validate that there are no other regions with a higher HPA already
916 * associated with @port
917 * - establish a region reference if one is not already present
918 *
919 * - additionally allocate a decoder instance that will host @cxlr on
920 * @port
921 *
922 * - pin the region reference by the endpoint
923 * - account for how many entries in @port's target list are needed to
924 * cover all of the added endpoints.
925 */
cxl_port_attach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)926 static int cxl_port_attach_region(struct cxl_port *port,
927 struct cxl_region *cxlr,
928 struct cxl_endpoint_decoder *cxled, int pos)
929 {
930 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
931 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
932 struct cxl_region_ref *cxl_rr;
933 bool nr_targets_inc = false;
934 struct cxl_decoder *cxld;
935 unsigned long index;
936 int rc = -EBUSY;
937
938 lockdep_assert_held_write(&cxl_region_rwsem);
939
940 cxl_rr = cxl_rr_load(port, cxlr);
941 if (cxl_rr) {
942 struct cxl_ep *ep_iter;
943 int found = 0;
944
945 /*
946 * Walk the existing endpoints that have been attached to
947 * @cxlr at @port and see if they share the same 'next' port
948 * in the downstream direction. I.e. endpoints that share common
949 * upstream switch.
950 */
951 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
952 if (ep_iter == ep)
953 continue;
954 if (ep_iter->next == ep->next) {
955 found++;
956 break;
957 }
958 }
959
960 /*
961 * New target port, or @port is an endpoint port that always
962 * accounts its own local decode as a target.
963 */
964 if (!found || !ep->next) {
965 cxl_rr->nr_targets++;
966 nr_targets_inc = true;
967 }
968 } else {
969 cxl_rr = alloc_region_ref(port, cxlr, cxled);
970 if (IS_ERR(cxl_rr)) {
971 dev_dbg(&cxlr->dev,
972 "%s: failed to allocate region reference\n",
973 dev_name(&port->dev));
974 return PTR_ERR(cxl_rr);
975 }
976 nr_targets_inc = true;
977
978 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
979 if (rc)
980 goto out_erase;
981 }
982 cxld = cxl_rr->decoder;
983
984 /*
985 * the number of targets should not exceed the target_count
986 * of the decoder
987 */
988 if (is_switch_decoder(&cxld->dev)) {
989 struct cxl_switch_decoder *cxlsd;
990
991 cxlsd = to_cxl_switch_decoder(&cxld->dev);
992 if (cxl_rr->nr_targets > cxlsd->nr_targets) {
993 dev_dbg(&cxlr->dev,
994 "%s:%s %s add: %s:%s @ %d overflows targets: %d\n",
995 dev_name(port->uport_dev), dev_name(&port->dev),
996 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
997 dev_name(&cxled->cxld.dev), pos,
998 cxlsd->nr_targets);
999 rc = -ENXIO;
1000 goto out_erase;
1001 }
1002 }
1003
1004 rc = cxl_rr_ep_add(cxl_rr, cxled);
1005 if (rc) {
1006 dev_dbg(&cxlr->dev,
1007 "%s: failed to track endpoint %s:%s reference\n",
1008 dev_name(&port->dev), dev_name(&cxlmd->dev),
1009 dev_name(&cxld->dev));
1010 goto out_erase;
1011 }
1012
1013 dev_dbg(&cxlr->dev,
1014 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1015 dev_name(port->uport_dev), dev_name(&port->dev),
1016 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1017 dev_name(&cxled->cxld.dev), pos,
1018 ep ? ep->next ? dev_name(ep->next->uport_dev) :
1019 dev_name(&cxlmd->dev) :
1020 "none",
1021 cxl_rr->nr_eps, cxl_rr->nr_targets);
1022
1023 return 0;
1024 out_erase:
1025 if (nr_targets_inc)
1026 cxl_rr->nr_targets--;
1027 if (cxl_rr->nr_eps == 0)
1028 free_region_ref(cxl_rr);
1029 return rc;
1030 }
1031
cxl_port_detach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1032 static void cxl_port_detach_region(struct cxl_port *port,
1033 struct cxl_region *cxlr,
1034 struct cxl_endpoint_decoder *cxled)
1035 {
1036 struct cxl_region_ref *cxl_rr;
1037 struct cxl_ep *ep = NULL;
1038
1039 lockdep_assert_held_write(&cxl_region_rwsem);
1040
1041 cxl_rr = cxl_rr_load(port, cxlr);
1042 if (!cxl_rr)
1043 return;
1044
1045 /*
1046 * Endpoint ports do not carry cxl_ep references, and they
1047 * never target more than one endpoint by definition
1048 */
1049 if (cxl_rr->decoder == &cxled->cxld)
1050 cxl_rr->nr_eps--;
1051 else
1052 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1053 if (ep) {
1054 struct cxl_ep *ep_iter;
1055 unsigned long index;
1056 int found = 0;
1057
1058 cxl_rr->nr_eps--;
1059 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1060 if (ep_iter->next == ep->next) {
1061 found++;
1062 break;
1063 }
1064 }
1065 if (!found)
1066 cxl_rr->nr_targets--;
1067 }
1068
1069 if (cxl_rr->nr_eps == 0)
1070 free_region_ref(cxl_rr);
1071 }
1072
check_last_peer(struct cxl_endpoint_decoder * cxled,struct cxl_ep * ep,struct cxl_region_ref * cxl_rr,int distance)1073 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1074 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1075 int distance)
1076 {
1077 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1078 struct cxl_region *cxlr = cxl_rr->region;
1079 struct cxl_region_params *p = &cxlr->params;
1080 struct cxl_endpoint_decoder *cxled_peer;
1081 struct cxl_port *port = cxl_rr->port;
1082 struct cxl_memdev *cxlmd_peer;
1083 struct cxl_ep *ep_peer;
1084 int pos = cxled->pos;
1085
1086 /*
1087 * If this position wants to share a dport with the last endpoint mapped
1088 * then that endpoint, at index 'position - distance', must also be
1089 * mapped by this dport.
1090 */
1091 if (pos < distance) {
1092 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1093 dev_name(port->uport_dev), dev_name(&port->dev),
1094 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1095 return -ENXIO;
1096 }
1097 cxled_peer = p->targets[pos - distance];
1098 cxlmd_peer = cxled_to_memdev(cxled_peer);
1099 ep_peer = cxl_ep_load(port, cxlmd_peer);
1100 if (ep->dport != ep_peer->dport) {
1101 dev_dbg(&cxlr->dev,
1102 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1103 dev_name(port->uport_dev), dev_name(&port->dev),
1104 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1105 dev_name(&cxlmd_peer->dev),
1106 dev_name(&cxled_peer->cxld.dev));
1107 return -ENXIO;
1108 }
1109
1110 return 0;
1111 }
1112
check_interleave_cap(struct cxl_decoder * cxld,int iw,int ig)1113 static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig)
1114 {
1115 struct cxl_port *port = to_cxl_port(cxld->dev.parent);
1116 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
1117 unsigned int interleave_mask;
1118 u8 eiw;
1119 u16 eig;
1120 int high_pos, low_pos;
1121
1122 if (!test_bit(iw, &cxlhdm->iw_cap_mask))
1123 return -ENXIO;
1124 /*
1125 * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection),
1126 * if eiw < 8:
1127 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw]
1128 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1129 *
1130 * when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the
1131 * interleave bits are none.
1132 *
1133 * if eiw >= 8:
1134 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3
1135 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0]
1136 *
1137 * when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the
1138 * interleave bits are none.
1139 */
1140 ways_to_eiw(iw, &eiw);
1141 if (eiw == 0 || eiw == 8)
1142 return 0;
1143
1144 granularity_to_eig(ig, &eig);
1145 if (eiw > 8)
1146 high_pos = eiw + eig - 1;
1147 else
1148 high_pos = eiw + eig + 7;
1149 low_pos = eig + 8;
1150 interleave_mask = GENMASK(high_pos, low_pos);
1151 if (interleave_mask & ~cxlhdm->interleave_mask)
1152 return -ENXIO;
1153
1154 return 0;
1155 }
1156
cxl_port_setup_targets(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1157 static int cxl_port_setup_targets(struct cxl_port *port,
1158 struct cxl_region *cxlr,
1159 struct cxl_endpoint_decoder *cxled)
1160 {
1161 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1162 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1163 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1164 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1165 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1166 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1167 struct cxl_region_params *p = &cxlr->params;
1168 struct cxl_decoder *cxld = cxl_rr->decoder;
1169 struct cxl_switch_decoder *cxlsd;
1170 struct cxl_port *iter = port;
1171 u16 eig, peig;
1172 u8 eiw, peiw;
1173
1174 /*
1175 * While root level decoders support x3, x6, x12, switch level
1176 * decoders only support powers of 2 up to x16.
1177 */
1178 if (!is_power_of_2(cxl_rr->nr_targets)) {
1179 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1180 dev_name(port->uport_dev), dev_name(&port->dev),
1181 cxl_rr->nr_targets);
1182 return -EINVAL;
1183 }
1184
1185 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1186 if (cxl_rr->nr_targets_set) {
1187 int i, distance = 1;
1188 struct cxl_region_ref *cxl_rr_iter;
1189
1190 /*
1191 * The "distance" between peer downstream ports represents which
1192 * endpoint positions in the region interleave a given port can
1193 * host.
1194 *
1195 * For example, at the root of a hierarchy the distance is
1196 * always 1 as every index targets a different host-bridge. At
1197 * each subsequent switch level those ports map every Nth region
1198 * position where N is the width of the switch == distance.
1199 */
1200 do {
1201 cxl_rr_iter = cxl_rr_load(iter, cxlr);
1202 distance *= cxl_rr_iter->nr_targets;
1203 iter = to_cxl_port(iter->dev.parent);
1204 } while (!is_cxl_root(iter));
1205 distance *= cxlrd->cxlsd.cxld.interleave_ways;
1206
1207 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1208 if (ep->dport == cxlsd->target[i]) {
1209 rc = check_last_peer(cxled, ep, cxl_rr,
1210 distance);
1211 if (rc)
1212 return rc;
1213 goto out_target_set;
1214 }
1215 goto add_target;
1216 }
1217
1218 if (is_cxl_root(parent_port)) {
1219 /*
1220 * Root decoder IG is always set to value in CFMWS which
1221 * may be different than this region's IG. We can use the
1222 * region's IG here since interleave_granularity_store()
1223 * does not allow interleaved host-bridges with
1224 * root IG != region IG.
1225 */
1226 parent_ig = p->interleave_granularity;
1227 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1228 /*
1229 * For purposes of address bit routing, use power-of-2 math for
1230 * switch ports.
1231 */
1232 if (!is_power_of_2(parent_iw))
1233 parent_iw /= 3;
1234 } else {
1235 struct cxl_region_ref *parent_rr;
1236 struct cxl_decoder *parent_cxld;
1237
1238 parent_rr = cxl_rr_load(parent_port, cxlr);
1239 parent_cxld = parent_rr->decoder;
1240 parent_ig = parent_cxld->interleave_granularity;
1241 parent_iw = parent_cxld->interleave_ways;
1242 }
1243
1244 rc = granularity_to_eig(parent_ig, &peig);
1245 if (rc) {
1246 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1247 dev_name(parent_port->uport_dev),
1248 dev_name(&parent_port->dev), parent_ig);
1249 return rc;
1250 }
1251
1252 rc = ways_to_eiw(parent_iw, &peiw);
1253 if (rc) {
1254 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1255 dev_name(parent_port->uport_dev),
1256 dev_name(&parent_port->dev), parent_iw);
1257 return rc;
1258 }
1259
1260 iw = cxl_rr->nr_targets;
1261 rc = ways_to_eiw(iw, &eiw);
1262 if (rc) {
1263 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1264 dev_name(port->uport_dev), dev_name(&port->dev), iw);
1265 return rc;
1266 }
1267
1268 /*
1269 * Interleave granularity is a multiple of @parent_port granularity.
1270 * Multiplier is the parent port interleave ways.
1271 */
1272 rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1273 if (rc) {
1274 dev_dbg(&cxlr->dev,
1275 "%s: invalid granularity calculation (%d * %d)\n",
1276 dev_name(&parent_port->dev), parent_ig, parent_iw);
1277 return rc;
1278 }
1279
1280 rc = eig_to_granularity(eig, &ig);
1281 if (rc) {
1282 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1283 dev_name(port->uport_dev), dev_name(&port->dev),
1284 256 << eig);
1285 return rc;
1286 }
1287
1288 if (iw > 8 || iw > cxlsd->nr_targets) {
1289 dev_dbg(&cxlr->dev,
1290 "%s:%s:%s: ways: %d overflows targets: %d\n",
1291 dev_name(port->uport_dev), dev_name(&port->dev),
1292 dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1293 return -ENXIO;
1294 }
1295
1296 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1297 if (cxld->interleave_ways != iw ||
1298 cxld->interleave_granularity != ig ||
1299 cxld->hpa_range.start != p->res->start ||
1300 cxld->hpa_range.end != p->res->end ||
1301 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1302 dev_err(&cxlr->dev,
1303 "%s:%s %s expected iw: %d ig: %d %pr\n",
1304 dev_name(port->uport_dev), dev_name(&port->dev),
1305 __func__, iw, ig, p->res);
1306 dev_err(&cxlr->dev,
1307 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1308 dev_name(port->uport_dev), dev_name(&port->dev),
1309 __func__, cxld->interleave_ways,
1310 cxld->interleave_granularity,
1311 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1312 "enabled" :
1313 "disabled",
1314 cxld->hpa_range.start, cxld->hpa_range.end);
1315 return -ENXIO;
1316 }
1317 } else {
1318 rc = check_interleave_cap(cxld, iw, ig);
1319 if (rc) {
1320 dev_dbg(&cxlr->dev,
1321 "%s:%s iw: %d ig: %d is not supported\n",
1322 dev_name(port->uport_dev),
1323 dev_name(&port->dev), iw, ig);
1324 return rc;
1325 }
1326
1327 cxld->interleave_ways = iw;
1328 cxld->interleave_granularity = ig;
1329 cxld->hpa_range = (struct range) {
1330 .start = p->res->start,
1331 .end = p->res->end,
1332 };
1333 }
1334 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1335 dev_name(&port->dev), iw, ig);
1336 add_target:
1337 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1338 dev_dbg(&cxlr->dev,
1339 "%s:%s: targets full trying to add %s:%s at %d\n",
1340 dev_name(port->uport_dev), dev_name(&port->dev),
1341 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1342 return -ENXIO;
1343 }
1344 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1345 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1346 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1347 dev_name(port->uport_dev), dev_name(&port->dev),
1348 dev_name(&cxlsd->cxld.dev),
1349 dev_name(ep->dport->dport_dev),
1350 cxl_rr->nr_targets_set);
1351 return -ENXIO;
1352 }
1353 } else
1354 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1355 inc = 1;
1356 out_target_set:
1357 cxl_rr->nr_targets_set += inc;
1358 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1359 dev_name(port->uport_dev), dev_name(&port->dev),
1360 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1361 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1362
1363 return 0;
1364 }
1365
cxl_port_reset_targets(struct cxl_port * port,struct cxl_region * cxlr)1366 static void cxl_port_reset_targets(struct cxl_port *port,
1367 struct cxl_region *cxlr)
1368 {
1369 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1370 struct cxl_decoder *cxld;
1371
1372 /*
1373 * After the last endpoint has been detached the entire cxl_rr may now
1374 * be gone.
1375 */
1376 if (!cxl_rr)
1377 return;
1378 cxl_rr->nr_targets_set = 0;
1379
1380 cxld = cxl_rr->decoder;
1381 cxld->hpa_range = (struct range) {
1382 .start = 0,
1383 .end = -1,
1384 };
1385 }
1386
cxl_region_teardown_targets(struct cxl_region * cxlr)1387 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1388 {
1389 struct cxl_region_params *p = &cxlr->params;
1390 struct cxl_endpoint_decoder *cxled;
1391 struct cxl_dev_state *cxlds;
1392 struct cxl_memdev *cxlmd;
1393 struct cxl_port *iter;
1394 struct cxl_ep *ep;
1395 int i;
1396
1397 /*
1398 * In the auto-discovery case skip automatic teardown since the
1399 * address space is already active
1400 */
1401 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1402 return;
1403
1404 for (i = 0; i < p->nr_targets; i++) {
1405 cxled = p->targets[i];
1406 cxlmd = cxled_to_memdev(cxled);
1407 cxlds = cxlmd->cxlds;
1408
1409 if (cxlds->rcd)
1410 continue;
1411
1412 iter = cxled_to_port(cxled);
1413 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1414 iter = to_cxl_port(iter->dev.parent);
1415
1416 for (ep = cxl_ep_load(iter, cxlmd); iter;
1417 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1418 cxl_port_reset_targets(iter, cxlr);
1419 }
1420 }
1421
cxl_region_setup_targets(struct cxl_region * cxlr)1422 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1423 {
1424 struct cxl_region_params *p = &cxlr->params;
1425 struct cxl_endpoint_decoder *cxled;
1426 struct cxl_dev_state *cxlds;
1427 int i, rc, rch = 0, vh = 0;
1428 struct cxl_memdev *cxlmd;
1429 struct cxl_port *iter;
1430 struct cxl_ep *ep;
1431
1432 for (i = 0; i < p->nr_targets; i++) {
1433 cxled = p->targets[i];
1434 cxlmd = cxled_to_memdev(cxled);
1435 cxlds = cxlmd->cxlds;
1436
1437 /* validate that all targets agree on topology */
1438 if (!cxlds->rcd) {
1439 vh++;
1440 } else {
1441 rch++;
1442 continue;
1443 }
1444
1445 iter = cxled_to_port(cxled);
1446 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1447 iter = to_cxl_port(iter->dev.parent);
1448
1449 /*
1450 * Descend the topology tree programming / validating
1451 * targets while looking for conflicts.
1452 */
1453 for (ep = cxl_ep_load(iter, cxlmd); iter;
1454 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1455 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1456 if (rc) {
1457 cxl_region_teardown_targets(cxlr);
1458 return rc;
1459 }
1460 }
1461 }
1462
1463 if (rch && vh) {
1464 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1465 cxl_region_teardown_targets(cxlr);
1466 return -ENXIO;
1467 }
1468
1469 return 0;
1470 }
1471
cxl_region_validate_position(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1472 static int cxl_region_validate_position(struct cxl_region *cxlr,
1473 struct cxl_endpoint_decoder *cxled,
1474 int pos)
1475 {
1476 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1477 struct cxl_region_params *p = &cxlr->params;
1478 int i;
1479
1480 if (pos < 0 || pos >= p->interleave_ways) {
1481 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1482 p->interleave_ways);
1483 return -ENXIO;
1484 }
1485
1486 if (p->targets[pos] == cxled)
1487 return 0;
1488
1489 if (p->targets[pos]) {
1490 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1491 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1492
1493 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1494 pos, dev_name(&cxlmd_target->dev),
1495 dev_name(&cxled_target->cxld.dev));
1496 return -EBUSY;
1497 }
1498
1499 for (i = 0; i < p->interleave_ways; i++) {
1500 struct cxl_endpoint_decoder *cxled_target;
1501 struct cxl_memdev *cxlmd_target;
1502
1503 cxled_target = p->targets[i];
1504 if (!cxled_target)
1505 continue;
1506
1507 cxlmd_target = cxled_to_memdev(cxled_target);
1508 if (cxlmd_target == cxlmd) {
1509 dev_dbg(&cxlr->dev,
1510 "%s already specified at position %d via: %s\n",
1511 dev_name(&cxlmd->dev), pos,
1512 dev_name(&cxled_target->cxld.dev));
1513 return -EBUSY;
1514 }
1515 }
1516
1517 return 0;
1518 }
1519
cxl_region_attach_position(struct cxl_region * cxlr,struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled,const struct cxl_dport * dport,int pos)1520 static int cxl_region_attach_position(struct cxl_region *cxlr,
1521 struct cxl_root_decoder *cxlrd,
1522 struct cxl_endpoint_decoder *cxled,
1523 const struct cxl_dport *dport, int pos)
1524 {
1525 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1526 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1527 struct cxl_decoder *cxld = &cxlsd->cxld;
1528 int iw = cxld->interleave_ways;
1529 struct cxl_port *iter;
1530 int rc;
1531
1532 if (dport != cxlrd->cxlsd.target[pos % iw]) {
1533 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1534 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1535 dev_name(&cxlrd->cxlsd.cxld.dev));
1536 return -ENXIO;
1537 }
1538
1539 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1540 iter = to_cxl_port(iter->dev.parent)) {
1541 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1542 if (rc)
1543 goto err;
1544 }
1545
1546 return 0;
1547
1548 err:
1549 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1550 iter = to_cxl_port(iter->dev.parent))
1551 cxl_port_detach_region(iter, cxlr, cxled);
1552 return rc;
1553 }
1554
cxl_region_attach_auto(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1555 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1556 struct cxl_endpoint_decoder *cxled, int pos)
1557 {
1558 struct cxl_region_params *p = &cxlr->params;
1559
1560 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1561 dev_err(&cxlr->dev,
1562 "%s: unable to add decoder to autodetected region\n",
1563 dev_name(&cxled->cxld.dev));
1564 return -EINVAL;
1565 }
1566
1567 if (pos >= 0) {
1568 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1569 dev_name(&cxled->cxld.dev), pos);
1570 return -EINVAL;
1571 }
1572
1573 if (p->nr_targets >= p->interleave_ways) {
1574 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1575 dev_name(&cxled->cxld.dev));
1576 return -ENXIO;
1577 }
1578
1579 /*
1580 * Temporarily record the endpoint decoder into the target array. Yes,
1581 * this means that userspace can view devices in the wrong position
1582 * before the region activates, and must be careful to understand when
1583 * it might be racing region autodiscovery.
1584 */
1585 pos = p->nr_targets;
1586 p->targets[pos] = cxled;
1587 cxled->pos = pos;
1588 p->nr_targets++;
1589
1590 return 0;
1591 }
1592
cmp_interleave_pos(const void * a,const void * b)1593 static int cmp_interleave_pos(const void *a, const void *b)
1594 {
1595 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1596 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1597
1598 return cxled_a->pos - cxled_b->pos;
1599 }
1600
next_port(struct cxl_port * port)1601 static struct cxl_port *next_port(struct cxl_port *port)
1602 {
1603 if (!port->parent_dport)
1604 return NULL;
1605 return port->parent_dport->port;
1606 }
1607
match_switch_decoder_by_range(struct device * dev,void * data)1608 static int match_switch_decoder_by_range(struct device *dev, void *data)
1609 {
1610 struct cxl_switch_decoder *cxlsd;
1611 struct range *r1, *r2 = data;
1612
1613 if (!is_switch_decoder(dev))
1614 return 0;
1615
1616 cxlsd = to_cxl_switch_decoder(dev);
1617 r1 = &cxlsd->cxld.hpa_range;
1618
1619 if (is_root_decoder(dev))
1620 return range_contains(r1, r2);
1621 return (r1->start == r2->start && r1->end == r2->end);
1622 }
1623
find_pos_and_ways(struct cxl_port * port,struct range * range,int * pos,int * ways)1624 static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1625 int *pos, int *ways)
1626 {
1627 struct cxl_switch_decoder *cxlsd;
1628 struct cxl_port *parent;
1629 struct device *dev;
1630 int rc = -ENXIO;
1631
1632 parent = next_port(port);
1633 if (!parent)
1634 return rc;
1635
1636 dev = device_find_child(&parent->dev, range,
1637 match_switch_decoder_by_range);
1638 if (!dev) {
1639 dev_err(port->uport_dev,
1640 "failed to find decoder mapping %#llx-%#llx\n",
1641 range->start, range->end);
1642 return rc;
1643 }
1644 cxlsd = to_cxl_switch_decoder(dev);
1645 *ways = cxlsd->cxld.interleave_ways;
1646
1647 for (int i = 0; i < *ways; i++) {
1648 if (cxlsd->target[i] == port->parent_dport) {
1649 *pos = i;
1650 rc = 0;
1651 break;
1652 }
1653 }
1654 put_device(dev);
1655
1656 return rc;
1657 }
1658
1659 /**
1660 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1661 * @cxled: endpoint decoder member of given region
1662 *
1663 * The endpoint position is calculated by traversing the topology from
1664 * the endpoint to the root decoder and iteratively applying this
1665 * calculation:
1666 *
1667 * position = position * parent_ways + parent_pos;
1668 *
1669 * ...where @position is inferred from switch and root decoder target lists.
1670 *
1671 * Return: position >= 0 on success
1672 * -ENXIO on failure
1673 */
cxl_calc_interleave_pos(struct cxl_endpoint_decoder * cxled)1674 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1675 {
1676 struct cxl_port *iter, *port = cxled_to_port(cxled);
1677 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1678 struct range *range = &cxled->cxld.hpa_range;
1679 int parent_ways = 0, parent_pos = 0, pos = 0;
1680 int rc;
1681
1682 /*
1683 * Example: the expected interleave order of the 4-way region shown
1684 * below is: mem0, mem2, mem1, mem3
1685 *
1686 * root_port
1687 * / \
1688 * host_bridge_0 host_bridge_1
1689 * | | | |
1690 * mem0 mem1 mem2 mem3
1691 *
1692 * In the example the calculator will iterate twice. The first iteration
1693 * uses the mem position in the host-bridge and the ways of the host-
1694 * bridge to generate the first, or local, position. The second
1695 * iteration uses the host-bridge position in the root_port and the ways
1696 * of the root_port to refine the position.
1697 *
1698 * A trace of the calculation per endpoint looks like this:
1699 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1700 * pos = 0 * 2 + 0 pos = 0 * 2 + 1
1701 * pos: 0 pos: 1
1702 *
1703 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1704 * pos = 1 * 2 + 0 pos = 1 * 2 + 1
1705 * pos: 2 pos = 3
1706 *
1707 * Note that while this example is simple, the method applies to more
1708 * complex topologies, including those with switches.
1709 */
1710
1711 /* Iterate from endpoint to root_port refining the position */
1712 for (iter = port; iter; iter = next_port(iter)) {
1713 if (is_cxl_root(iter))
1714 break;
1715
1716 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1717 if (rc)
1718 return rc;
1719
1720 pos = pos * parent_ways + parent_pos;
1721 }
1722
1723 dev_dbg(&cxlmd->dev,
1724 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1725 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1726 dev_name(&port->dev), range->start, range->end, pos);
1727
1728 return pos;
1729 }
1730
cxl_region_sort_targets(struct cxl_region * cxlr)1731 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1732 {
1733 struct cxl_region_params *p = &cxlr->params;
1734 int i, rc = 0;
1735
1736 for (i = 0; i < p->nr_targets; i++) {
1737 struct cxl_endpoint_decoder *cxled = p->targets[i];
1738
1739 cxled->pos = cxl_calc_interleave_pos(cxled);
1740 /*
1741 * Record that sorting failed, but still continue to calc
1742 * cxled->pos so that follow-on code paths can reliably
1743 * do p->targets[cxled->pos] to self-reference their entry.
1744 */
1745 if (cxled->pos < 0)
1746 rc = -ENXIO;
1747 }
1748 /* Keep the cxlr target list in interleave position order */
1749 sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1750 cmp_interleave_pos, NULL);
1751
1752 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1753 return rc;
1754 }
1755
cxl_region_attach(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1756 static int cxl_region_attach(struct cxl_region *cxlr,
1757 struct cxl_endpoint_decoder *cxled, int pos)
1758 {
1759 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1760 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1761 struct cxl_region_params *p = &cxlr->params;
1762 struct cxl_port *ep_port, *root_port;
1763 struct cxl_dport *dport;
1764 int rc = -ENXIO;
1765
1766 rc = check_interleave_cap(&cxled->cxld, p->interleave_ways,
1767 p->interleave_granularity);
1768 if (rc) {
1769 dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n",
1770 dev_name(&cxled->cxld.dev), p->interleave_ways,
1771 p->interleave_granularity);
1772 return rc;
1773 }
1774
1775 if (cxled->mode != cxlr->mode) {
1776 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1777 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1778 return -EINVAL;
1779 }
1780
1781 if (cxled->mode == CXL_DECODER_DEAD) {
1782 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1783 return -ENODEV;
1784 }
1785
1786 /* all full of members, or interleave config not established? */
1787 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1788 dev_dbg(&cxlr->dev, "region already active\n");
1789 return -EBUSY;
1790 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1791 dev_dbg(&cxlr->dev, "interleave config missing\n");
1792 return -ENXIO;
1793 }
1794
1795 if (p->nr_targets >= p->interleave_ways) {
1796 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1797 p->nr_targets);
1798 return -EINVAL;
1799 }
1800
1801 ep_port = cxled_to_port(cxled);
1802 root_port = cxlrd_to_port(cxlrd);
1803 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1804 if (!dport) {
1805 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1806 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1807 dev_name(cxlr->dev.parent));
1808 return -ENXIO;
1809 }
1810
1811 if (cxled->cxld.target_type != cxlr->type) {
1812 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1813 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1814 cxled->cxld.target_type, cxlr->type);
1815 return -ENXIO;
1816 }
1817
1818 if (!cxled->dpa_res) {
1819 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1820 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1821 return -ENXIO;
1822 }
1823
1824 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1825 resource_size(p->res)) {
1826 dev_dbg(&cxlr->dev,
1827 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1828 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1829 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1830 (u64)resource_size(p->res));
1831 return -EINVAL;
1832 }
1833
1834 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1835 int i;
1836
1837 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1838 if (rc)
1839 return rc;
1840
1841 /* await more targets to arrive... */
1842 if (p->nr_targets < p->interleave_ways)
1843 return 0;
1844
1845 /*
1846 * All targets are here, which implies all PCI enumeration that
1847 * affects this region has been completed. Walk the topology to
1848 * sort the devices into their relative region decode position.
1849 */
1850 rc = cxl_region_sort_targets(cxlr);
1851 if (rc)
1852 return rc;
1853
1854 for (i = 0; i < p->nr_targets; i++) {
1855 cxled = p->targets[i];
1856 ep_port = cxled_to_port(cxled);
1857 dport = cxl_find_dport_by_dev(root_port,
1858 ep_port->host_bridge);
1859 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1860 dport, i);
1861 if (rc)
1862 return rc;
1863 }
1864
1865 rc = cxl_region_setup_targets(cxlr);
1866 if (rc)
1867 return rc;
1868
1869 /*
1870 * If target setup succeeds in the autodiscovery case
1871 * then the region is already committed.
1872 */
1873 p->state = CXL_CONFIG_COMMIT;
1874
1875 return 0;
1876 }
1877
1878 rc = cxl_region_validate_position(cxlr, cxled, pos);
1879 if (rc)
1880 return rc;
1881
1882 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1883 if (rc)
1884 return rc;
1885
1886 p->targets[pos] = cxled;
1887 cxled->pos = pos;
1888 p->nr_targets++;
1889
1890 if (p->nr_targets == p->interleave_ways) {
1891 rc = cxl_region_setup_targets(cxlr);
1892 if (rc)
1893 return rc;
1894 p->state = CXL_CONFIG_ACTIVE;
1895 }
1896
1897 cxled->cxld.interleave_ways = p->interleave_ways;
1898 cxled->cxld.interleave_granularity = p->interleave_granularity;
1899 cxled->cxld.hpa_range = (struct range) {
1900 .start = p->res->start,
1901 .end = p->res->end,
1902 };
1903
1904 if (p->nr_targets != p->interleave_ways)
1905 return 0;
1906
1907 /*
1908 * Test the auto-discovery position calculator function
1909 * against this successfully created user-defined region.
1910 * A fail message here means that this interleave config
1911 * will fail when presented as CXL_REGION_F_AUTO.
1912 */
1913 for (int i = 0; i < p->nr_targets; i++) {
1914 struct cxl_endpoint_decoder *cxled = p->targets[i];
1915 int test_pos;
1916
1917 test_pos = cxl_calc_interleave_pos(cxled);
1918 dev_dbg(&cxled->cxld.dev,
1919 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
1920 (test_pos == cxled->pos) ? "success" : "fail",
1921 test_pos, cxled->pos);
1922 }
1923
1924 return 0;
1925 }
1926
cxl_region_detach(struct cxl_endpoint_decoder * cxled)1927 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1928 {
1929 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1930 struct cxl_region *cxlr = cxled->cxld.region;
1931 struct cxl_region_params *p;
1932 int rc = 0;
1933
1934 lockdep_assert_held_write(&cxl_region_rwsem);
1935
1936 if (!cxlr)
1937 return 0;
1938
1939 p = &cxlr->params;
1940 get_device(&cxlr->dev);
1941
1942 if (p->state > CXL_CONFIG_ACTIVE) {
1943 cxl_region_decode_reset(cxlr, p->interleave_ways);
1944 p->state = CXL_CONFIG_ACTIVE;
1945 }
1946
1947 for (iter = ep_port; !is_cxl_root(iter);
1948 iter = to_cxl_port(iter->dev.parent))
1949 cxl_port_detach_region(iter, cxlr, cxled);
1950
1951 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1952 p->targets[cxled->pos] != cxled) {
1953 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1954
1955 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1956 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1957 cxled->pos);
1958 goto out;
1959 }
1960
1961 if (p->state == CXL_CONFIG_ACTIVE) {
1962 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1963 cxl_region_teardown_targets(cxlr);
1964 }
1965 p->targets[cxled->pos] = NULL;
1966 p->nr_targets--;
1967 cxled->cxld.hpa_range = (struct range) {
1968 .start = 0,
1969 .end = -1,
1970 };
1971
1972 /* notify the region driver that one of its targets has departed */
1973 up_write(&cxl_region_rwsem);
1974 device_release_driver(&cxlr->dev);
1975 down_write(&cxl_region_rwsem);
1976 out:
1977 put_device(&cxlr->dev);
1978 return rc;
1979 }
1980
cxl_decoder_kill_region(struct cxl_endpoint_decoder * cxled)1981 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1982 {
1983 down_write(&cxl_region_rwsem);
1984 cxled->mode = CXL_DECODER_DEAD;
1985 cxl_region_detach(cxled);
1986 up_write(&cxl_region_rwsem);
1987 }
1988
attach_target(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos,unsigned int state)1989 static int attach_target(struct cxl_region *cxlr,
1990 struct cxl_endpoint_decoder *cxled, int pos,
1991 unsigned int state)
1992 {
1993 int rc = 0;
1994
1995 if (state == TASK_INTERRUPTIBLE)
1996 rc = down_write_killable(&cxl_region_rwsem);
1997 else
1998 down_write(&cxl_region_rwsem);
1999 if (rc)
2000 return rc;
2001
2002 down_read(&cxl_dpa_rwsem);
2003 rc = cxl_region_attach(cxlr, cxled, pos);
2004 up_read(&cxl_dpa_rwsem);
2005 up_write(&cxl_region_rwsem);
2006 return rc;
2007 }
2008
detach_target(struct cxl_region * cxlr,int pos)2009 static int detach_target(struct cxl_region *cxlr, int pos)
2010 {
2011 struct cxl_region_params *p = &cxlr->params;
2012 int rc;
2013
2014 rc = down_write_killable(&cxl_region_rwsem);
2015 if (rc)
2016 return rc;
2017
2018 if (pos >= p->interleave_ways) {
2019 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
2020 p->interleave_ways);
2021 rc = -ENXIO;
2022 goto out;
2023 }
2024
2025 if (!p->targets[pos]) {
2026 rc = 0;
2027 goto out;
2028 }
2029
2030 rc = cxl_region_detach(p->targets[pos]);
2031 out:
2032 up_write(&cxl_region_rwsem);
2033 return rc;
2034 }
2035
store_targetN(struct cxl_region * cxlr,const char * buf,int pos,size_t len)2036 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
2037 size_t len)
2038 {
2039 int rc;
2040
2041 if (sysfs_streq(buf, "\n"))
2042 rc = detach_target(cxlr, pos);
2043 else {
2044 struct device *dev;
2045
2046 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
2047 if (!dev)
2048 return -ENODEV;
2049
2050 if (!is_endpoint_decoder(dev)) {
2051 rc = -EINVAL;
2052 goto out;
2053 }
2054
2055 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
2056 TASK_INTERRUPTIBLE);
2057 out:
2058 put_device(dev);
2059 }
2060
2061 if (rc < 0)
2062 return rc;
2063 return len;
2064 }
2065
2066 #define TARGET_ATTR_RW(n) \
2067 static ssize_t target##n##_show( \
2068 struct device *dev, struct device_attribute *attr, char *buf) \
2069 { \
2070 return show_targetN(to_cxl_region(dev), buf, (n)); \
2071 } \
2072 static ssize_t target##n##_store(struct device *dev, \
2073 struct device_attribute *attr, \
2074 const char *buf, size_t len) \
2075 { \
2076 return store_targetN(to_cxl_region(dev), buf, (n), len); \
2077 } \
2078 static DEVICE_ATTR_RW(target##n)
2079
2080 TARGET_ATTR_RW(0);
2081 TARGET_ATTR_RW(1);
2082 TARGET_ATTR_RW(2);
2083 TARGET_ATTR_RW(3);
2084 TARGET_ATTR_RW(4);
2085 TARGET_ATTR_RW(5);
2086 TARGET_ATTR_RW(6);
2087 TARGET_ATTR_RW(7);
2088 TARGET_ATTR_RW(8);
2089 TARGET_ATTR_RW(9);
2090 TARGET_ATTR_RW(10);
2091 TARGET_ATTR_RW(11);
2092 TARGET_ATTR_RW(12);
2093 TARGET_ATTR_RW(13);
2094 TARGET_ATTR_RW(14);
2095 TARGET_ATTR_RW(15);
2096
2097 static struct attribute *target_attrs[] = {
2098 &dev_attr_target0.attr,
2099 &dev_attr_target1.attr,
2100 &dev_attr_target2.attr,
2101 &dev_attr_target3.attr,
2102 &dev_attr_target4.attr,
2103 &dev_attr_target5.attr,
2104 &dev_attr_target6.attr,
2105 &dev_attr_target7.attr,
2106 &dev_attr_target8.attr,
2107 &dev_attr_target9.attr,
2108 &dev_attr_target10.attr,
2109 &dev_attr_target11.attr,
2110 &dev_attr_target12.attr,
2111 &dev_attr_target13.attr,
2112 &dev_attr_target14.attr,
2113 &dev_attr_target15.attr,
2114 NULL,
2115 };
2116
cxl_region_target_visible(struct kobject * kobj,struct attribute * a,int n)2117 static umode_t cxl_region_target_visible(struct kobject *kobj,
2118 struct attribute *a, int n)
2119 {
2120 struct device *dev = kobj_to_dev(kobj);
2121 struct cxl_region *cxlr = to_cxl_region(dev);
2122 struct cxl_region_params *p = &cxlr->params;
2123
2124 if (n < p->interleave_ways)
2125 return a->mode;
2126 return 0;
2127 }
2128
2129 static const struct attribute_group cxl_region_target_group = {
2130 .attrs = target_attrs,
2131 .is_visible = cxl_region_target_visible,
2132 };
2133
get_cxl_region_target_group(void)2134 static const struct attribute_group *get_cxl_region_target_group(void)
2135 {
2136 return &cxl_region_target_group;
2137 }
2138
2139 static const struct attribute_group *region_groups[] = {
2140 &cxl_base_attribute_group,
2141 &cxl_region_group,
2142 &cxl_region_target_group,
2143 NULL,
2144 };
2145
cxl_region_release(struct device * dev)2146 static void cxl_region_release(struct device *dev)
2147 {
2148 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2149 struct cxl_region *cxlr = to_cxl_region(dev);
2150 int id = atomic_read(&cxlrd->region_id);
2151
2152 /*
2153 * Try to reuse the recently idled id rather than the cached
2154 * next id to prevent the region id space from increasing
2155 * unnecessarily.
2156 */
2157 if (cxlr->id < id)
2158 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2159 memregion_free(id);
2160 goto out;
2161 }
2162
2163 memregion_free(cxlr->id);
2164 out:
2165 put_device(dev->parent);
2166 kfree(cxlr);
2167 }
2168
2169 const struct device_type cxl_region_type = {
2170 .name = "cxl_region",
2171 .release = cxl_region_release,
2172 .groups = region_groups
2173 };
2174
is_cxl_region(struct device * dev)2175 bool is_cxl_region(struct device *dev)
2176 {
2177 return dev->type == &cxl_region_type;
2178 }
2179 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2180
to_cxl_region(struct device * dev)2181 static struct cxl_region *to_cxl_region(struct device *dev)
2182 {
2183 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2184 "not a cxl_region device\n"))
2185 return NULL;
2186
2187 return container_of(dev, struct cxl_region, dev);
2188 }
2189
unregister_region(void * dev)2190 static void unregister_region(void *dev)
2191 {
2192 struct cxl_region *cxlr = to_cxl_region(dev);
2193 struct cxl_region_params *p = &cxlr->params;
2194 int i;
2195
2196 device_del(dev);
2197
2198 /*
2199 * Now that region sysfs is shutdown, the parameter block is now
2200 * read-only, so no need to hold the region rwsem to access the
2201 * region parameters.
2202 */
2203 for (i = 0; i < p->interleave_ways; i++)
2204 detach_target(cxlr, i);
2205
2206 cxl_region_iomem_release(cxlr);
2207 put_device(dev);
2208 }
2209
2210 static struct lock_class_key cxl_region_key;
2211
cxl_region_alloc(struct cxl_root_decoder * cxlrd,int id)2212 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2213 {
2214 struct cxl_region *cxlr;
2215 struct device *dev;
2216
2217 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2218 if (!cxlr) {
2219 memregion_free(id);
2220 return ERR_PTR(-ENOMEM);
2221 }
2222
2223 dev = &cxlr->dev;
2224 device_initialize(dev);
2225 lockdep_set_class(&dev->mutex, &cxl_region_key);
2226 dev->parent = &cxlrd->cxlsd.cxld.dev;
2227 /*
2228 * Keep root decoder pinned through cxl_region_release to fixup
2229 * region id allocations
2230 */
2231 get_device(dev->parent);
2232 device_set_pm_not_required(dev);
2233 dev->bus = &cxl_bus_type;
2234 dev->type = &cxl_region_type;
2235 cxlr->id = id;
2236
2237 return cxlr;
2238 }
2239
2240 /**
2241 * devm_cxl_add_region - Adds a region to a decoder
2242 * @cxlrd: root decoder
2243 * @id: memregion id to create, or memregion_free() on failure
2244 * @mode: mode for the endpoint decoders of this region
2245 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2246 *
2247 * This is the second step of region initialization. Regions exist within an
2248 * address space which is mapped by a @cxlrd.
2249 *
2250 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2251 * code. The region will be named "regionZ" where Z is the unique region number.
2252 */
devm_cxl_add_region(struct cxl_root_decoder * cxlrd,int id,enum cxl_decoder_mode mode,enum cxl_decoder_type type)2253 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2254 int id,
2255 enum cxl_decoder_mode mode,
2256 enum cxl_decoder_type type)
2257 {
2258 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2259 struct cxl_region *cxlr;
2260 struct device *dev;
2261 int rc;
2262
2263 cxlr = cxl_region_alloc(cxlrd, id);
2264 if (IS_ERR(cxlr))
2265 return cxlr;
2266 cxlr->mode = mode;
2267 cxlr->type = type;
2268
2269 dev = &cxlr->dev;
2270 rc = dev_set_name(dev, "region%d", id);
2271 if (rc)
2272 goto err;
2273
2274 rc = device_add(dev);
2275 if (rc)
2276 goto err;
2277
2278 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2279 if (rc)
2280 return ERR_PTR(rc);
2281
2282 dev_dbg(port->uport_dev, "%s: created %s\n",
2283 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2284 return cxlr;
2285
2286 err:
2287 put_device(dev);
2288 return ERR_PTR(rc);
2289 }
2290
__create_region_show(struct cxl_root_decoder * cxlrd,char * buf)2291 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2292 {
2293 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2294 }
2295
create_pmem_region_show(struct device * dev,struct device_attribute * attr,char * buf)2296 static ssize_t create_pmem_region_show(struct device *dev,
2297 struct device_attribute *attr, char *buf)
2298 {
2299 return __create_region_show(to_cxl_root_decoder(dev), buf);
2300 }
2301
create_ram_region_show(struct device * dev,struct device_attribute * attr,char * buf)2302 static ssize_t create_ram_region_show(struct device *dev,
2303 struct device_attribute *attr, char *buf)
2304 {
2305 return __create_region_show(to_cxl_root_decoder(dev), buf);
2306 }
2307
__create_region(struct cxl_root_decoder * cxlrd,enum cxl_decoder_mode mode,int id)2308 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2309 enum cxl_decoder_mode mode, int id)
2310 {
2311 int rc;
2312
2313 switch (mode) {
2314 case CXL_DECODER_RAM:
2315 case CXL_DECODER_PMEM:
2316 break;
2317 default:
2318 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2319 return ERR_PTR(-EINVAL);
2320 }
2321
2322 rc = memregion_alloc(GFP_KERNEL);
2323 if (rc < 0)
2324 return ERR_PTR(rc);
2325
2326 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2327 memregion_free(rc);
2328 return ERR_PTR(-EBUSY);
2329 }
2330
2331 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2332 }
2333
create_pmem_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2334 static ssize_t create_pmem_region_store(struct device *dev,
2335 struct device_attribute *attr,
2336 const char *buf, size_t len)
2337 {
2338 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2339 struct cxl_region *cxlr;
2340 int rc, id;
2341
2342 rc = sscanf(buf, "region%d\n", &id);
2343 if (rc != 1)
2344 return -EINVAL;
2345
2346 cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2347 if (IS_ERR(cxlr))
2348 return PTR_ERR(cxlr);
2349
2350 return len;
2351 }
2352 DEVICE_ATTR_RW(create_pmem_region);
2353
create_ram_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2354 static ssize_t create_ram_region_store(struct device *dev,
2355 struct device_attribute *attr,
2356 const char *buf, size_t len)
2357 {
2358 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2359 struct cxl_region *cxlr;
2360 int rc, id;
2361
2362 rc = sscanf(buf, "region%d\n", &id);
2363 if (rc != 1)
2364 return -EINVAL;
2365
2366 cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2367 if (IS_ERR(cxlr))
2368 return PTR_ERR(cxlr);
2369
2370 return len;
2371 }
2372 DEVICE_ATTR_RW(create_ram_region);
2373
region_show(struct device * dev,struct device_attribute * attr,char * buf)2374 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2375 char *buf)
2376 {
2377 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2378 ssize_t rc;
2379
2380 rc = down_read_interruptible(&cxl_region_rwsem);
2381 if (rc)
2382 return rc;
2383
2384 if (cxld->region)
2385 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2386 else
2387 rc = sysfs_emit(buf, "\n");
2388 up_read(&cxl_region_rwsem);
2389
2390 return rc;
2391 }
2392 DEVICE_ATTR_RO(region);
2393
2394 static struct cxl_region *
cxl_find_region_by_name(struct cxl_root_decoder * cxlrd,const char * name)2395 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2396 {
2397 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2398 struct device *region_dev;
2399
2400 region_dev = device_find_child_by_name(&cxld->dev, name);
2401 if (!region_dev)
2402 return ERR_PTR(-ENODEV);
2403
2404 return to_cxl_region(region_dev);
2405 }
2406
delete_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2407 static ssize_t delete_region_store(struct device *dev,
2408 struct device_attribute *attr,
2409 const char *buf, size_t len)
2410 {
2411 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2412 struct cxl_port *port = to_cxl_port(dev->parent);
2413 struct cxl_region *cxlr;
2414
2415 cxlr = cxl_find_region_by_name(cxlrd, buf);
2416 if (IS_ERR(cxlr))
2417 return PTR_ERR(cxlr);
2418
2419 devm_release_action(port->uport_dev, unregister_region, cxlr);
2420 put_device(&cxlr->dev);
2421
2422 return len;
2423 }
2424 DEVICE_ATTR_WO(delete_region);
2425
cxl_pmem_region_release(struct device * dev)2426 static void cxl_pmem_region_release(struct device *dev)
2427 {
2428 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2429 int i;
2430
2431 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2432 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2433
2434 put_device(&cxlmd->dev);
2435 }
2436
2437 kfree(cxlr_pmem);
2438 }
2439
2440 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2441 &cxl_base_attribute_group,
2442 NULL,
2443 };
2444
2445 const struct device_type cxl_pmem_region_type = {
2446 .name = "cxl_pmem_region",
2447 .release = cxl_pmem_region_release,
2448 .groups = cxl_pmem_region_attribute_groups,
2449 };
2450
is_cxl_pmem_region(struct device * dev)2451 bool is_cxl_pmem_region(struct device *dev)
2452 {
2453 return dev->type == &cxl_pmem_region_type;
2454 }
2455 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2456
to_cxl_pmem_region(struct device * dev)2457 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2458 {
2459 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2460 "not a cxl_pmem_region device\n"))
2461 return NULL;
2462 return container_of(dev, struct cxl_pmem_region, dev);
2463 }
2464 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2465
2466 struct cxl_poison_context {
2467 struct cxl_port *port;
2468 enum cxl_decoder_mode mode;
2469 u64 offset;
2470 };
2471
cxl_get_poison_unmapped(struct cxl_memdev * cxlmd,struct cxl_poison_context * ctx)2472 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2473 struct cxl_poison_context *ctx)
2474 {
2475 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2476 u64 offset, length;
2477 int rc = 0;
2478
2479 /*
2480 * Collect poison for the remaining unmapped resources
2481 * after poison is collected by committed endpoints.
2482 *
2483 * Knowing that PMEM must always follow RAM, get poison
2484 * for unmapped resources based on the last decoder's mode:
2485 * ram: scan remains of ram range, then any pmem range
2486 * pmem: scan remains of pmem range
2487 */
2488
2489 if (ctx->mode == CXL_DECODER_RAM) {
2490 offset = ctx->offset;
2491 length = resource_size(&cxlds->ram_res) - offset;
2492 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2493 if (rc == -EFAULT)
2494 rc = 0;
2495 if (rc)
2496 return rc;
2497 }
2498 if (ctx->mode == CXL_DECODER_PMEM) {
2499 offset = ctx->offset;
2500 length = resource_size(&cxlds->dpa_res) - offset;
2501 if (!length)
2502 return 0;
2503 } else if (resource_size(&cxlds->pmem_res)) {
2504 offset = cxlds->pmem_res.start;
2505 length = resource_size(&cxlds->pmem_res);
2506 } else {
2507 return 0;
2508 }
2509
2510 return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2511 }
2512
poison_by_decoder(struct device * dev,void * arg)2513 static int poison_by_decoder(struct device *dev, void *arg)
2514 {
2515 struct cxl_poison_context *ctx = arg;
2516 struct cxl_endpoint_decoder *cxled;
2517 struct cxl_memdev *cxlmd;
2518 u64 offset, length;
2519 int rc = 0;
2520
2521 if (!is_endpoint_decoder(dev))
2522 return rc;
2523
2524 cxled = to_cxl_endpoint_decoder(dev);
2525 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2526 return rc;
2527
2528 /*
2529 * Regions are only created with single mode decoders: pmem or ram.
2530 * Linux does not support mixed mode decoders. This means that
2531 * reading poison per endpoint decoder adheres to the requirement
2532 * that poison reads of pmem and ram must be separated.
2533 * CXL 3.0 Spec 8.2.9.8.4.1
2534 */
2535 if (cxled->mode == CXL_DECODER_MIXED) {
2536 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2537 return rc;
2538 }
2539
2540 cxlmd = cxled_to_memdev(cxled);
2541 if (cxled->skip) {
2542 offset = cxled->dpa_res->start - cxled->skip;
2543 length = cxled->skip;
2544 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2545 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2546 rc = 0;
2547 if (rc)
2548 return rc;
2549 }
2550
2551 offset = cxled->dpa_res->start;
2552 length = cxled->dpa_res->end - offset + 1;
2553 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2554 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2555 rc = 0;
2556 if (rc)
2557 return rc;
2558
2559 /* Iterate until commit_end is reached */
2560 if (cxled->cxld.id == ctx->port->commit_end) {
2561 ctx->offset = cxled->dpa_res->end + 1;
2562 ctx->mode = cxled->mode;
2563 return 1;
2564 }
2565
2566 return 0;
2567 }
2568
cxl_get_poison_by_endpoint(struct cxl_port * port)2569 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2570 {
2571 struct cxl_poison_context ctx;
2572 int rc = 0;
2573
2574 ctx = (struct cxl_poison_context) {
2575 .port = port
2576 };
2577
2578 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2579 if (rc == 1)
2580 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2581 &ctx);
2582
2583 return rc;
2584 }
2585
2586 struct cxl_dpa_to_region_context {
2587 struct cxl_region *cxlr;
2588 u64 dpa;
2589 };
2590
__cxl_dpa_to_region(struct device * dev,void * arg)2591 static int __cxl_dpa_to_region(struct device *dev, void *arg)
2592 {
2593 struct cxl_dpa_to_region_context *ctx = arg;
2594 struct cxl_endpoint_decoder *cxled;
2595 struct cxl_region *cxlr;
2596 u64 dpa = ctx->dpa;
2597
2598 if (!is_endpoint_decoder(dev))
2599 return 0;
2600
2601 cxled = to_cxl_endpoint_decoder(dev);
2602 if (!cxled || !cxled->dpa_res || !resource_size(cxled->dpa_res))
2603 return 0;
2604
2605 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
2606 return 0;
2607
2608 /*
2609 * Stop the region search (return 1) when an endpoint mapping is
2610 * found. The region may not be fully constructed so offering
2611 * the cxlr in the context structure is not guaranteed.
2612 */
2613 cxlr = cxled->cxld.region;
2614 if (cxlr)
2615 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
2616 dev_name(&cxlr->dev));
2617 else
2618 dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa,
2619 dev_name(dev));
2620
2621 ctx->cxlr = cxlr;
2622
2623 return 1;
2624 }
2625
cxl_dpa_to_region(const struct cxl_memdev * cxlmd,u64 dpa)2626 struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
2627 {
2628 struct cxl_dpa_to_region_context ctx;
2629 struct cxl_port *port;
2630
2631 ctx = (struct cxl_dpa_to_region_context) {
2632 .dpa = dpa,
2633 };
2634 port = cxlmd->endpoint;
2635 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
2636 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
2637
2638 return ctx.cxlr;
2639 }
2640
2641 static struct lock_class_key cxl_pmem_region_key;
2642
cxl_pmem_region_alloc(struct cxl_region * cxlr)2643 static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2644 {
2645 struct cxl_region_params *p = &cxlr->params;
2646 struct cxl_nvdimm_bridge *cxl_nvb;
2647 struct cxl_pmem_region *cxlr_pmem;
2648 struct device *dev;
2649 int i;
2650
2651 down_read(&cxl_region_rwsem);
2652 if (p->state != CXL_CONFIG_COMMIT) {
2653 cxlr_pmem = ERR_PTR(-ENXIO);
2654 goto out;
2655 }
2656
2657 cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2658 GFP_KERNEL);
2659 if (!cxlr_pmem) {
2660 cxlr_pmem = ERR_PTR(-ENOMEM);
2661 goto out;
2662 }
2663
2664 cxlr_pmem->hpa_range.start = p->res->start;
2665 cxlr_pmem->hpa_range.end = p->res->end;
2666
2667 /* Snapshot the region configuration underneath the cxl_region_rwsem */
2668 cxlr_pmem->nr_mappings = p->nr_targets;
2669 for (i = 0; i < p->nr_targets; i++) {
2670 struct cxl_endpoint_decoder *cxled = p->targets[i];
2671 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2672 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2673
2674 /*
2675 * Regions never span CXL root devices, so by definition the
2676 * bridge for one device is the same for all.
2677 */
2678 if (i == 0) {
2679 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2680 if (!cxl_nvb) {
2681 kfree(cxlr_pmem);
2682 cxlr_pmem = ERR_PTR(-ENODEV);
2683 goto out;
2684 }
2685 cxlr->cxl_nvb = cxl_nvb;
2686 }
2687 m->cxlmd = cxlmd;
2688 get_device(&cxlmd->dev);
2689 m->start = cxled->dpa_res->start;
2690 m->size = resource_size(cxled->dpa_res);
2691 m->position = i;
2692 }
2693
2694 dev = &cxlr_pmem->dev;
2695 cxlr_pmem->cxlr = cxlr;
2696 cxlr->cxlr_pmem = cxlr_pmem;
2697 device_initialize(dev);
2698 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2699 device_set_pm_not_required(dev);
2700 dev->parent = &cxlr->dev;
2701 dev->bus = &cxl_bus_type;
2702 dev->type = &cxl_pmem_region_type;
2703 out:
2704 up_read(&cxl_region_rwsem);
2705
2706 return cxlr_pmem;
2707 }
2708
cxl_dax_region_release(struct device * dev)2709 static void cxl_dax_region_release(struct device *dev)
2710 {
2711 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2712
2713 kfree(cxlr_dax);
2714 }
2715
2716 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2717 &cxl_base_attribute_group,
2718 NULL,
2719 };
2720
2721 const struct device_type cxl_dax_region_type = {
2722 .name = "cxl_dax_region",
2723 .release = cxl_dax_region_release,
2724 .groups = cxl_dax_region_attribute_groups,
2725 };
2726
is_cxl_dax_region(struct device * dev)2727 static bool is_cxl_dax_region(struct device *dev)
2728 {
2729 return dev->type == &cxl_dax_region_type;
2730 }
2731
to_cxl_dax_region(struct device * dev)2732 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2733 {
2734 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2735 "not a cxl_dax_region device\n"))
2736 return NULL;
2737 return container_of(dev, struct cxl_dax_region, dev);
2738 }
2739 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2740
2741 static struct lock_class_key cxl_dax_region_key;
2742
cxl_dax_region_alloc(struct cxl_region * cxlr)2743 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2744 {
2745 struct cxl_region_params *p = &cxlr->params;
2746 struct cxl_dax_region *cxlr_dax;
2747 struct device *dev;
2748
2749 down_read(&cxl_region_rwsem);
2750 if (p->state != CXL_CONFIG_COMMIT) {
2751 cxlr_dax = ERR_PTR(-ENXIO);
2752 goto out;
2753 }
2754
2755 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2756 if (!cxlr_dax) {
2757 cxlr_dax = ERR_PTR(-ENOMEM);
2758 goto out;
2759 }
2760
2761 cxlr_dax->hpa_range.start = p->res->start;
2762 cxlr_dax->hpa_range.end = p->res->end;
2763
2764 dev = &cxlr_dax->dev;
2765 cxlr_dax->cxlr = cxlr;
2766 device_initialize(dev);
2767 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2768 device_set_pm_not_required(dev);
2769 dev->parent = &cxlr->dev;
2770 dev->bus = &cxl_bus_type;
2771 dev->type = &cxl_dax_region_type;
2772 out:
2773 up_read(&cxl_region_rwsem);
2774
2775 return cxlr_dax;
2776 }
2777
cxlr_pmem_unregister(void * _cxlr_pmem)2778 static void cxlr_pmem_unregister(void *_cxlr_pmem)
2779 {
2780 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2781 struct cxl_region *cxlr = cxlr_pmem->cxlr;
2782 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2783
2784 /*
2785 * Either the bridge is in ->remove() context under the device_lock(),
2786 * or cxlr_release_nvdimm() is cancelling the bridge's release action
2787 * for @cxlr_pmem and doing it itself (while manually holding the bridge
2788 * lock).
2789 */
2790 device_lock_assert(&cxl_nvb->dev);
2791 cxlr->cxlr_pmem = NULL;
2792 cxlr_pmem->cxlr = NULL;
2793 device_unregister(&cxlr_pmem->dev);
2794 }
2795
cxlr_release_nvdimm(void * _cxlr)2796 static void cxlr_release_nvdimm(void *_cxlr)
2797 {
2798 struct cxl_region *cxlr = _cxlr;
2799 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2800
2801 device_lock(&cxl_nvb->dev);
2802 if (cxlr->cxlr_pmem)
2803 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2804 cxlr->cxlr_pmem);
2805 device_unlock(&cxl_nvb->dev);
2806 cxlr->cxl_nvb = NULL;
2807 put_device(&cxl_nvb->dev);
2808 }
2809
2810 /**
2811 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2812 * @cxlr: parent CXL region for this pmem region bridge device
2813 *
2814 * Return: 0 on success negative error code on failure.
2815 */
devm_cxl_add_pmem_region(struct cxl_region * cxlr)2816 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2817 {
2818 struct cxl_pmem_region *cxlr_pmem;
2819 struct cxl_nvdimm_bridge *cxl_nvb;
2820 struct device *dev;
2821 int rc;
2822
2823 cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2824 if (IS_ERR(cxlr_pmem))
2825 return PTR_ERR(cxlr_pmem);
2826 cxl_nvb = cxlr->cxl_nvb;
2827
2828 dev = &cxlr_pmem->dev;
2829 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2830 if (rc)
2831 goto err;
2832
2833 rc = device_add(dev);
2834 if (rc)
2835 goto err;
2836
2837 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2838 dev_name(dev));
2839
2840 device_lock(&cxl_nvb->dev);
2841 if (cxl_nvb->dev.driver)
2842 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2843 cxlr_pmem_unregister, cxlr_pmem);
2844 else
2845 rc = -ENXIO;
2846 device_unlock(&cxl_nvb->dev);
2847
2848 if (rc)
2849 goto err_bridge;
2850
2851 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2852 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
2853
2854 err:
2855 put_device(dev);
2856 err_bridge:
2857 put_device(&cxl_nvb->dev);
2858 cxlr->cxl_nvb = NULL;
2859 return rc;
2860 }
2861
cxlr_dax_unregister(void * _cxlr_dax)2862 static void cxlr_dax_unregister(void *_cxlr_dax)
2863 {
2864 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2865
2866 device_unregister(&cxlr_dax->dev);
2867 }
2868
devm_cxl_add_dax_region(struct cxl_region * cxlr)2869 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2870 {
2871 struct cxl_dax_region *cxlr_dax;
2872 struct device *dev;
2873 int rc;
2874
2875 cxlr_dax = cxl_dax_region_alloc(cxlr);
2876 if (IS_ERR(cxlr_dax))
2877 return PTR_ERR(cxlr_dax);
2878
2879 dev = &cxlr_dax->dev;
2880 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2881 if (rc)
2882 goto err;
2883
2884 rc = device_add(dev);
2885 if (rc)
2886 goto err;
2887
2888 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2889 dev_name(dev));
2890
2891 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2892 cxlr_dax);
2893 err:
2894 put_device(dev);
2895 return rc;
2896 }
2897
match_root_decoder_by_range(struct device * dev,void * data)2898 static int match_root_decoder_by_range(struct device *dev, void *data)
2899 {
2900 struct range *r1, *r2 = data;
2901 struct cxl_root_decoder *cxlrd;
2902
2903 if (!is_root_decoder(dev))
2904 return 0;
2905
2906 cxlrd = to_cxl_root_decoder(dev);
2907 r1 = &cxlrd->cxlsd.cxld.hpa_range;
2908 return range_contains(r1, r2);
2909 }
2910
match_region_by_range(struct device * dev,void * data)2911 static int match_region_by_range(struct device *dev, void *data)
2912 {
2913 struct cxl_region_params *p;
2914 struct cxl_region *cxlr;
2915 struct range *r = data;
2916 int rc = 0;
2917
2918 if (!is_cxl_region(dev))
2919 return 0;
2920
2921 cxlr = to_cxl_region(dev);
2922 p = &cxlr->params;
2923
2924 down_read(&cxl_region_rwsem);
2925 if (p->res && p->res->start == r->start && p->res->end == r->end)
2926 rc = 1;
2927 up_read(&cxl_region_rwsem);
2928
2929 return rc;
2930 }
2931
2932 /* Establish an empty region covering the given HPA range */
construct_region(struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled)2933 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2934 struct cxl_endpoint_decoder *cxled)
2935 {
2936 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2937 struct cxl_port *port = cxlrd_to_port(cxlrd);
2938 struct range *hpa = &cxled->cxld.hpa_range;
2939 struct cxl_region_params *p;
2940 struct cxl_region *cxlr;
2941 struct resource *res;
2942 int rc;
2943
2944 do {
2945 cxlr = __create_region(cxlrd, cxled->mode,
2946 atomic_read(&cxlrd->region_id));
2947 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2948
2949 if (IS_ERR(cxlr)) {
2950 dev_err(cxlmd->dev.parent,
2951 "%s:%s: %s failed assign region: %ld\n",
2952 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2953 __func__, PTR_ERR(cxlr));
2954 return cxlr;
2955 }
2956
2957 down_write(&cxl_region_rwsem);
2958 p = &cxlr->params;
2959 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2960 dev_err(cxlmd->dev.parent,
2961 "%s:%s: %s autodiscovery interrupted\n",
2962 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2963 __func__);
2964 rc = -EBUSY;
2965 goto err;
2966 }
2967
2968 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2969
2970 res = kmalloc(sizeof(*res), GFP_KERNEL);
2971 if (!res) {
2972 rc = -ENOMEM;
2973 goto err;
2974 }
2975
2976 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2977 dev_name(&cxlr->dev));
2978 rc = insert_resource(cxlrd->res, res);
2979 if (rc) {
2980 /*
2981 * Platform-firmware may not have split resources like "System
2982 * RAM" on CXL window boundaries see cxl_region_iomem_release()
2983 */
2984 dev_warn(cxlmd->dev.parent,
2985 "%s:%s: %s %s cannot insert resource\n",
2986 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2987 __func__, dev_name(&cxlr->dev));
2988 }
2989
2990 p->res = res;
2991 p->interleave_ways = cxled->cxld.interleave_ways;
2992 p->interleave_granularity = cxled->cxld.interleave_granularity;
2993 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2994
2995 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2996 if (rc)
2997 goto err;
2998
2999 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
3000 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
3001 dev_name(&cxlr->dev), p->res, p->interleave_ways,
3002 p->interleave_granularity);
3003
3004 /* ...to match put_device() in cxl_add_to_region() */
3005 get_device(&cxlr->dev);
3006 up_write(&cxl_region_rwsem);
3007
3008 return cxlr;
3009
3010 err:
3011 up_write(&cxl_region_rwsem);
3012 devm_release_action(port->uport_dev, unregister_region, cxlr);
3013 return ERR_PTR(rc);
3014 }
3015
cxl_add_to_region(struct cxl_port * root,struct cxl_endpoint_decoder * cxled)3016 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
3017 {
3018 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3019 struct range *hpa = &cxled->cxld.hpa_range;
3020 struct cxl_decoder *cxld = &cxled->cxld;
3021 struct device *cxlrd_dev, *region_dev;
3022 struct cxl_root_decoder *cxlrd;
3023 struct cxl_region_params *p;
3024 struct cxl_region *cxlr;
3025 bool attach = false;
3026 int rc;
3027
3028 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
3029 match_root_decoder_by_range);
3030 if (!cxlrd_dev) {
3031 dev_err(cxlmd->dev.parent,
3032 "%s:%s no CXL window for range %#llx:%#llx\n",
3033 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
3034 cxld->hpa_range.start, cxld->hpa_range.end);
3035 return -ENXIO;
3036 }
3037
3038 cxlrd = to_cxl_root_decoder(cxlrd_dev);
3039
3040 /*
3041 * Ensure that if multiple threads race to construct_region() for @hpa
3042 * one does the construction and the others add to that.
3043 */
3044 mutex_lock(&cxlrd->range_lock);
3045 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
3046 match_region_by_range);
3047 if (!region_dev) {
3048 cxlr = construct_region(cxlrd, cxled);
3049 region_dev = &cxlr->dev;
3050 } else
3051 cxlr = to_cxl_region(region_dev);
3052 mutex_unlock(&cxlrd->range_lock);
3053
3054 rc = PTR_ERR_OR_ZERO(cxlr);
3055 if (rc)
3056 goto out;
3057
3058 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
3059
3060 down_read(&cxl_region_rwsem);
3061 p = &cxlr->params;
3062 attach = p->state == CXL_CONFIG_COMMIT;
3063 up_read(&cxl_region_rwsem);
3064
3065 if (attach) {
3066 /*
3067 * If device_attach() fails the range may still be active via
3068 * the platform-firmware memory map, otherwise the driver for
3069 * regions is local to this file, so driver matching can't fail.
3070 */
3071 if (device_attach(&cxlr->dev) < 0)
3072 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
3073 p->res);
3074 }
3075
3076 put_device(region_dev);
3077 out:
3078 put_device(cxlrd_dev);
3079 return rc;
3080 }
3081 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
3082
is_system_ram(struct resource * res,void * arg)3083 static int is_system_ram(struct resource *res, void *arg)
3084 {
3085 struct cxl_region *cxlr = arg;
3086 struct cxl_region_params *p = &cxlr->params;
3087
3088 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
3089 return 1;
3090 }
3091
cxl_region_probe(struct device * dev)3092 static int cxl_region_probe(struct device *dev)
3093 {
3094 struct cxl_region *cxlr = to_cxl_region(dev);
3095 struct cxl_region_params *p = &cxlr->params;
3096 int rc;
3097
3098 rc = down_read_interruptible(&cxl_region_rwsem);
3099 if (rc) {
3100 dev_dbg(&cxlr->dev, "probe interrupted\n");
3101 return rc;
3102 }
3103
3104 if (p->state < CXL_CONFIG_COMMIT) {
3105 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
3106 rc = -ENXIO;
3107 goto out;
3108 }
3109
3110 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
3111 dev_err(&cxlr->dev,
3112 "failed to activate, re-commit region and retry\n");
3113 rc = -ENXIO;
3114 goto out;
3115 }
3116
3117 /*
3118 * From this point on any path that changes the region's state away from
3119 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
3120 */
3121 out:
3122 up_read(&cxl_region_rwsem);
3123
3124 if (rc)
3125 return rc;
3126
3127 switch (cxlr->mode) {
3128 case CXL_DECODER_PMEM:
3129 return devm_cxl_add_pmem_region(cxlr);
3130 case CXL_DECODER_RAM:
3131 /*
3132 * The region can not be manged by CXL if any portion of
3133 * it is already online as 'System RAM'
3134 */
3135 if (walk_iomem_res_desc(IORES_DESC_NONE,
3136 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3137 p->res->start, p->res->end, cxlr,
3138 is_system_ram) > 0)
3139 return 0;
3140 return devm_cxl_add_dax_region(cxlr);
3141 default:
3142 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3143 cxlr->mode);
3144 return -ENXIO;
3145 }
3146 }
3147
3148 static struct cxl_driver cxl_region_driver = {
3149 .name = "cxl_region",
3150 .probe = cxl_region_probe,
3151 .id = CXL_DEVICE_REGION,
3152 };
3153
cxl_region_init(void)3154 int cxl_region_init(void)
3155 {
3156 return cxl_driver_register(&cxl_region_driver);
3157 }
3158
cxl_region_exit(void)3159 void cxl_region_exit(void)
3160 {
3161 cxl_driver_unregister(&cxl_region_driver);
3162 }
3163
3164 MODULE_IMPORT_NS(CXL);
3165 MODULE_IMPORT_NS(DEVMEM);
3166 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
3167