native.c (368857c16c595eb7537cc0846708ddaa57a3a25b) native.c (0b3f9c757cabad4b8101c5fcddddd029ed5506a6)
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */

--- 27 unchanged lines hidden (view full) ---

36
37 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
38 while ((AFU_Cntl & mask) != result) {
39 if (time_after_eq(jiffies, timeout)) {
40 dev_warn(&afu->dev, "WARNING: AFU control timed out!\n");
41 rc = -EBUSY;
42 goto out;
43 }
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */

--- 27 unchanged lines hidden (view full) ---

36
37 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
38 while ((AFU_Cntl & mask) != result) {
39 if (time_after_eq(jiffies, timeout)) {
40 dev_warn(&afu->dev, "WARNING: AFU control timed out!\n");
41 rc = -EBUSY;
42 goto out;
43 }
44
45 if (!cxl_adapter_link_ok(afu->adapter)) {
46 afu->enabled = enabled;
47 rc = -EIO;
48 goto out;
49 }
50
44 pr_devel_ratelimited("AFU control... (0x%016llx)\n",
45 AFU_Cntl | command);
46 cpu_relax();
47 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
48 };
49 pr_devel("AFU command complete: %llx\n", command);
50 afu->enabled = enabled;
51out:

--- 28 unchanged lines hidden (view full) ---

80 return afu_control(afu, CXL_AFU_Cntl_An_RA,
81 CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled,
82 CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK,
83 false);
84}
85
86int cxl_afu_check_and_enable(struct cxl_afu *afu)
87{
51 pr_devel_ratelimited("AFU control... (0x%016llx)\n",
52 AFU_Cntl | command);
53 cpu_relax();
54 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
55 };
56 pr_devel("AFU command complete: %llx\n", command);
57 afu->enabled = enabled;
58out:

--- 28 unchanged lines hidden (view full) ---

87 return afu_control(afu, CXL_AFU_Cntl_An_RA,
88 CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled,
89 CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK,
90 false);
91}
92
93int cxl_afu_check_and_enable(struct cxl_afu *afu)
94{
95 if (!cxl_adapter_link_ok(afu->adapter)) {
96 WARN(1, "Refusing to enable afu while link down!\n");
97 return -EIO;
98 }
88 if (afu->enabled)
89 return 0;
90 return afu_enable(afu);
91}
92
93int cxl_psl_purge(struct cxl_afu *afu)
94{
95 u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
96 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
97 u64 dsisr, dar;
98 u64 start, end;
99 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
100 int rc = 0;
101
102 trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc);
103
104 pr_devel("PSL purge request\n");
105
99 if (afu->enabled)
100 return 0;
101 return afu_enable(afu);
102}
103
104int cxl_psl_purge(struct cxl_afu *afu)
105{
106 u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
107 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
108 u64 dsisr, dar;
109 u64 start, end;
110 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
111 int rc = 0;
112
113 trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc);
114
115 pr_devel("PSL purge request\n");
116
117 if (!cxl_adapter_link_ok(afu->adapter)) {
118 dev_warn(&afu->dev, "PSL Purge called with link down, ignoring\n");
119 rc = -EIO;
120 goto out;
121 }
122
106 if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
107 WARN(1, "psl_purge request while AFU not disabled!\n");
108 cxl_afu_disable(afu);
109 }
110
111 cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
112 PSL_CNTL | CXL_PSL_SCNTL_An_Pc);
113 start = local_clock();
114 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
115 while ((PSL_CNTL & CXL_PSL_SCNTL_An_Ps_MASK)
116 == CXL_PSL_SCNTL_An_Ps_Pending) {
117 if (time_after_eq(jiffies, timeout)) {
118 dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n");
119 rc = -EBUSY;
120 goto out;
121 }
123 if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
124 WARN(1, "psl_purge request while AFU not disabled!\n");
125 cxl_afu_disable(afu);
126 }
127
128 cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
129 PSL_CNTL | CXL_PSL_SCNTL_An_Pc);
130 start = local_clock();
131 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
132 while ((PSL_CNTL & CXL_PSL_SCNTL_An_Ps_MASK)
133 == CXL_PSL_SCNTL_An_Ps_Pending) {
134 if (time_after_eq(jiffies, timeout)) {
135 dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n");
136 rc = -EBUSY;
137 goto out;
138 }
139 if (!cxl_adapter_link_ok(afu->adapter)) {
140 rc = -EIO;
141 goto out;
142 }
143
122 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
123 pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%016llx PSL_DSISR: 0x%016llx\n", PSL_CNTL, dsisr);
124 if (dsisr & CXL_PSL_DSISR_TRANS) {
125 dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
126 dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n", dsisr, dar);
127 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
128 } else if (dsisr) {
129 dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n", dsisr);

--- 80 unchanged lines hidden (view full) ---

210 cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A);
211
212 cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL);
213 while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) {
214 if (time_after_eq(jiffies, timeout)) {
215 dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n");
216 return -EBUSY;
217 }
144 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
145 pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%016llx PSL_DSISR: 0x%016llx\n", PSL_CNTL, dsisr);
146 if (dsisr & CXL_PSL_DSISR_TRANS) {
147 dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
148 dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx\n", dsisr, dar);
149 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
150 } else if (dsisr) {
151 dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n", dsisr);

--- 80 unchanged lines hidden (view full) ---

232 cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A);
233
234 cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL);
235 while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) {
236 if (time_after_eq(jiffies, timeout)) {
237 dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n");
238 return -EBUSY;
239 }
240 if (!cxl_adapter_link_ok(adapter))
241 return -EIO;
218 cpu_relax();
219 }
220
221 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL);
222 while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) {
223 if (time_after_eq(jiffies, timeout)) {
224 dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n");
225 return -EBUSY;
226 }
242 cpu_relax();
243 }
244
245 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL);
246 while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) {
247 if (time_after_eq(jiffies, timeout)) {
248 dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n");
249 return -EBUSY;
250 }
251 if (!cxl_adapter_link_ok(adapter))
252 return -EIO;
227 cpu_relax();
228 }
229 return 0;
230}
231
232int cxl_afu_slbia(struct cxl_afu *afu)
233{
234 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
235
236 pr_devel("cxl_afu_slbia issuing SLBIA command\n");
237 cxl_p2n_write(afu, CXL_SLBIA_An, CXL_TLB_SLB_IQ_ALL);
238 while (cxl_p2n_read(afu, CXL_SLBIA_An) & CXL_TLB_SLB_P) {
239 if (time_after_eq(jiffies, timeout)) {
240 dev_warn(&afu->dev, "WARNING: CXL AFU SLBIA timed out!\n");
241 return -EBUSY;
242 }
253 cpu_relax();
254 }
255 return 0;
256}
257
258int cxl_afu_slbia(struct cxl_afu *afu)
259{
260 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
261
262 pr_devel("cxl_afu_slbia issuing SLBIA command\n");
263 cxl_p2n_write(afu, CXL_SLBIA_An, CXL_TLB_SLB_IQ_ALL);
264 while (cxl_p2n_read(afu, CXL_SLBIA_An) & CXL_TLB_SLB_P) {
265 if (time_after_eq(jiffies, timeout)) {
266 dev_warn(&afu->dev, "WARNING: CXL AFU SLBIA timed out!\n");
267 return -EBUSY;
268 }
269 /* If the adapter has gone down, we can assume that we
270 * will PERST it and that will invalidate everything.
271 */
272 if (!cxl_adapter_link_ok(afu->adapter))
273 return -EIO;
243 cpu_relax();
244 }
245 return 0;
246}
247
248static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
249{
250 int rc;

--- 23 unchanged lines hidden (view full) ---

274 WARN_ON(!mutex_is_locked(&ctx->afu->spa_mutex));
275
276 cxl_p1_write(adapter, CXL_PSL_LBISEL,
277 ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) |
278 be32_to_cpu(ctx->elem->lpid));
279 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID);
280
281 while (1) {
274 cpu_relax();
275 }
276 return 0;
277}
278
279static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
280{
281 int rc;

--- 23 unchanged lines hidden (view full) ---

305 WARN_ON(!mutex_is_locked(&ctx->afu->spa_mutex));
306
307 cxl_p1_write(adapter, CXL_PSL_LBISEL,
308 ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) |
309 be32_to_cpu(ctx->elem->lpid));
310 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID);
311
312 while (1) {
313 if (!cxl_adapter_link_ok(adapter))
314 break;
282 slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA);
283 if (!(slbia & CXL_TLB_SLB_P))
284 break;
285 cpu_relax();
286 }
287}
288
289static int do_process_element_cmd(struct cxl_context *ctx,

--- 13 unchanged lines hidden (view full) ---

303 smp_mb();
304 cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe);
305 while (1) {
306 if (time_after_eq(jiffies, timeout)) {
307 dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n");
308 rc = -EBUSY;
309 goto out;
310 }
315 slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA);
316 if (!(slbia & CXL_TLB_SLB_P))
317 break;
318 cpu_relax();
319 }
320}
321
322static int do_process_element_cmd(struct cxl_context *ctx,

--- 13 unchanged lines hidden (view full) ---

336 smp_mb();
337 cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe);
338 while (1) {
339 if (time_after_eq(jiffies, timeout)) {
340 dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n");
341 rc = -EBUSY;
342 goto out;
343 }
344 if (!cxl_adapter_link_ok(ctx->afu->adapter)) {
345 dev_warn(&ctx->afu->dev, "WARNING: Device link down, aborting Process Element Command!\n");
346 rc = -EIO;
347 goto out;
348 }
311 state = be64_to_cpup(ctx->afu->sw_command_status);
312 if (state == ~0ULL) {
313 pr_err("cxl: Error adding process element to AFU\n");
314 rc = -1;
315 goto out;
316 }
317 if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK | CXL_SPA_SW_LINK_MASK)) ==
318 (cmd | (cmd >> 16) | ctx->pe))

--- 31 unchanged lines hidden (view full) ---

350 int rc = 0;
351
352 /* fast path terminate if it's already invalid */
353 if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V)))
354 return rc;
355
356 mutex_lock(&ctx->afu->spa_mutex);
357 pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe);
349 state = be64_to_cpup(ctx->afu->sw_command_status);
350 if (state == ~0ULL) {
351 pr_err("cxl: Error adding process element to AFU\n");
352 rc = -1;
353 goto out;
354 }
355 if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK | CXL_SPA_SW_LINK_MASK)) ==
356 (cmd | (cmd >> 16) | ctx->pe))

--- 31 unchanged lines hidden (view full) ---

388 int rc = 0;
389
390 /* fast path terminate if it's already invalid */
391 if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V)))
392 return rc;
393
394 mutex_lock(&ctx->afu->spa_mutex);
395 pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe);
358 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE,
359 CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T);
396 /* We could be asked to terminate when the hw is down. That
397 * should always succeed: it's not running if the hw has gone
398 * away and is being reset.
399 */
400 if (cxl_adapter_link_ok(ctx->afu->adapter))
401 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE,
402 CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T);
360 ctx->elem->software_state = 0; /* Remove Valid bit */
361 pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe);
362 mutex_unlock(&ctx->afu->spa_mutex);
363 return rc;
364}
365
366static int remove_process_element(struct cxl_context *ctx)
367{
368 int rc = 0;
369
370 mutex_lock(&ctx->afu->spa_mutex);
371 pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe);
403 ctx->elem->software_state = 0; /* Remove Valid bit */
404 pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe);
405 mutex_unlock(&ctx->afu->spa_mutex);
406 return rc;
407}
408
409static int remove_process_element(struct cxl_context *ctx)
410{
411 int rc = 0;
412
413 mutex_lock(&ctx->afu->spa_mutex);
414 pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe);
372 if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0)))
415
416 /* We could be asked to remove when the hw is down. Again, if
417 * the hw is down, the PE is gone, so we succeed.
418 */
419 if (cxl_adapter_link_ok(ctx->afu->adapter))
420 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0);
421
422 if (!rc)
373 ctx->pe_inserted = false;
374 slb_invalid(ctx);
375 pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe);
376 mutex_unlock(&ctx->afu->spa_mutex);
377
378 return rc;
379}
380

--- 226 unchanged lines hidden (view full) ---

607
608int cxl_afu_activate_mode(struct cxl_afu *afu, int mode)
609{
610 if (!mode)
611 return 0;
612 if (!(mode & afu->modes_supported))
613 return -EINVAL;
614
423 ctx->pe_inserted = false;
424 slb_invalid(ctx);
425 pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe);
426 mutex_unlock(&ctx->afu->spa_mutex);
427
428 return rc;
429}
430

--- 226 unchanged lines hidden (view full) ---

657
658int cxl_afu_activate_mode(struct cxl_afu *afu, int mode)
659{
660 if (!mode)
661 return 0;
662 if (!(mode & afu->modes_supported))
663 return -EINVAL;
664
665 if (!cxl_adapter_link_ok(afu->adapter)) {
666 WARN(1, "Device link is down, refusing to activate!\n");
667 return -EIO;
668 }
669
615 if (mode == CXL_MODE_DIRECTED)
616 return activate_afu_directed(afu);
617 if (mode == CXL_MODE_DEDICATED)
618 return activate_dedicated_process(afu);
619
620 return -EINVAL;
621}
622
623int cxl_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr)
624{
670 if (mode == CXL_MODE_DIRECTED)
671 return activate_afu_directed(afu);
672 if (mode == CXL_MODE_DEDICATED)
673 return activate_dedicated_process(afu);
674
675 return -EINVAL;
676}
677
678int cxl_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr)
679{
680 if (!cxl_adapter_link_ok(ctx->afu->adapter)) {
681 WARN(1, "Device link is down, refusing to attach process!\n");
682 return -EIO;
683 }
684
625 ctx->kernel = kernel;
626 if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
627 return attach_afu_directed(ctx, wed, amr);
628
629 if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
630 return attach_dedicated(ctx, wed, amr);
631
632 return -EINVAL;

--- 28 unchanged lines hidden (view full) ---

661
662 return detach_process_native_afu_directed(ctx);
663}
664
665int cxl_get_irq(struct cxl_afu *afu, struct cxl_irq_info *info)
666{
667 u64 pidtid;
668
685 ctx->kernel = kernel;
686 if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
687 return attach_afu_directed(ctx, wed, amr);
688
689 if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
690 return attach_dedicated(ctx, wed, amr);
691
692 return -EINVAL;

--- 28 unchanged lines hidden (view full) ---

721
722 return detach_process_native_afu_directed(ctx);
723}
724
725int cxl_get_irq(struct cxl_afu *afu, struct cxl_irq_info *info)
726{
727 u64 pidtid;
728
729 /* If the adapter has gone away, we can't get any meaningful
730 * information.
731 */
732 if (!cxl_adapter_link_ok(afu->adapter))
733 return -EIO;
734
669 info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
670 info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
671 info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An);
672 pidtid = cxl_p2n_read(afu, CXL_PSL_PID_TID_An);
673 info->pid = pidtid >> 32;
674 info->tid = pidtid & 0xffffffff;
675 info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An);
676 info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);

--- 33 unchanged lines hidden --- 		735 info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
736 info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
737 info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An);
738 pidtid = cxl_p2n_read(afu, CXL_PSL_PID_TID_An);
739 info->pid = pidtid >> 32;
740 info->tid = pidtid & 0xffffffff;
741 info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An);
742 info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);

--- 33 unchanged lines hidden ---