1f204e0b8SIan Munsie /* 2f204e0b8SIan Munsie * Copyright 2014 IBM Corp. 3f204e0b8SIan Munsie * 4f204e0b8SIan Munsie * This program is free software; you can redistribute it and/or 5f204e0b8SIan Munsie * modify it under the terms of the GNU General Public License 6f204e0b8SIan Munsie * as published by the Free Software Foundation; either version 7f204e0b8SIan Munsie * 2 of the License, or (at your option) any later version. 8f204e0b8SIan Munsie */ 9f204e0b8SIan Munsie 10f204e0b8SIan Munsie #include <linux/spinlock.h> 11f204e0b8SIan Munsie #include <linux/sched.h> 12f204e0b8SIan Munsie #include <linux/slab.h> 13f204e0b8SIan Munsie #include <linux/sched.h> 14f204e0b8SIan Munsie #include <linux/mutex.h> 15f204e0b8SIan Munsie #include <linux/mm.h> 16f204e0b8SIan Munsie #include <linux/uaccess.h> 17f204e0b8SIan Munsie #include <asm/synch.h> 18f204e0b8SIan Munsie #include <misc/cxl.h> 19f204e0b8SIan Munsie 20f204e0b8SIan Munsie #include "cxl.h" 21f204e0b8SIan Munsie 22f204e0b8SIan Munsie static int afu_control(struct cxl_afu *afu, u64 command, 23f204e0b8SIan Munsie u64 result, u64 mask, bool enabled) 24f204e0b8SIan Munsie { 25f204e0b8SIan Munsie u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 26f204e0b8SIan Munsie unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 27f204e0b8SIan Munsie 28f204e0b8SIan Munsie spin_lock(&afu->afu_cntl_lock); 29f204e0b8SIan Munsie pr_devel("AFU command starting: %llx\n", command); 30f204e0b8SIan Munsie 31f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl | command); 32f204e0b8SIan Munsie 33f204e0b8SIan Munsie AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 34f204e0b8SIan Munsie while ((AFU_Cntl & mask) != result) { 35f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 36f204e0b8SIan Munsie dev_warn(&afu->dev, "WARNING: AFU control timed out!\n"); 37f204e0b8SIan Munsie spin_unlock(&afu->afu_cntl_lock); 38f204e0b8SIan Munsie return -EBUSY; 39f204e0b8SIan Munsie } 40f204e0b8SIan Munsie pr_devel_ratelimited("AFU control... (0x%.16llx)\n", 41f204e0b8SIan Munsie AFU_Cntl | command); 42f204e0b8SIan Munsie cpu_relax(); 43f204e0b8SIan Munsie AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 44f204e0b8SIan Munsie }; 45f204e0b8SIan Munsie pr_devel("AFU command complete: %llx\n", command); 46f204e0b8SIan Munsie afu->enabled = enabled; 47f204e0b8SIan Munsie spin_unlock(&afu->afu_cntl_lock); 48f204e0b8SIan Munsie 49f204e0b8SIan Munsie return 0; 50f204e0b8SIan Munsie } 51f204e0b8SIan Munsie 52f204e0b8SIan Munsie static int afu_enable(struct cxl_afu *afu) 53f204e0b8SIan Munsie { 54f204e0b8SIan Munsie pr_devel("AFU enable request\n"); 55f204e0b8SIan Munsie 56f204e0b8SIan Munsie return afu_control(afu, CXL_AFU_Cntl_An_E, 57f204e0b8SIan Munsie CXL_AFU_Cntl_An_ES_Enabled, 58f204e0b8SIan Munsie CXL_AFU_Cntl_An_ES_MASK, true); 59f204e0b8SIan Munsie } 60f204e0b8SIan Munsie 61f204e0b8SIan Munsie int cxl_afu_disable(struct cxl_afu *afu) 62f204e0b8SIan Munsie { 63f204e0b8SIan Munsie pr_devel("AFU disable request\n"); 64f204e0b8SIan Munsie 65f204e0b8SIan Munsie return afu_control(afu, 0, CXL_AFU_Cntl_An_ES_Disabled, 66f204e0b8SIan Munsie CXL_AFU_Cntl_An_ES_MASK, false); 67f204e0b8SIan Munsie } 68f204e0b8SIan Munsie 69f204e0b8SIan Munsie /* This will disable as well as reset */ 70f204e0b8SIan Munsie int cxl_afu_reset(struct cxl_afu *afu) 71f204e0b8SIan Munsie { 72f204e0b8SIan Munsie pr_devel("AFU reset request\n"); 73f204e0b8SIan Munsie 74f204e0b8SIan Munsie return afu_control(afu, CXL_AFU_Cntl_An_RA, 75f204e0b8SIan Munsie CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled, 76f204e0b8SIan Munsie CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK, 77f204e0b8SIan Munsie false); 78f204e0b8SIan Munsie } 79f204e0b8SIan Munsie 80f204e0b8SIan Munsie static int afu_check_and_enable(struct cxl_afu *afu) 81f204e0b8SIan Munsie { 82f204e0b8SIan Munsie if (afu->enabled) 83f204e0b8SIan Munsie return 0; 84f204e0b8SIan Munsie return afu_enable(afu); 85f204e0b8SIan Munsie } 86f204e0b8SIan Munsie 87f204e0b8SIan Munsie int cxl_psl_purge(struct cxl_afu *afu) 88f204e0b8SIan Munsie { 89f204e0b8SIan Munsie u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 90f204e0b8SIan Munsie u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 91f204e0b8SIan Munsie u64 dsisr, dar; 92f204e0b8SIan Munsie u64 start, end; 93f204e0b8SIan Munsie unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 94f204e0b8SIan Munsie 95f204e0b8SIan Munsie pr_devel("PSL purge request\n"); 96f204e0b8SIan Munsie 97f204e0b8SIan Munsie if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) { 98f204e0b8SIan Munsie WARN(1, "psl_purge request while AFU not disabled!\n"); 99f204e0b8SIan Munsie cxl_afu_disable(afu); 100f204e0b8SIan Munsie } 101f204e0b8SIan Munsie 102f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SCNTL_An, 103f204e0b8SIan Munsie PSL_CNTL | CXL_PSL_SCNTL_An_Pc); 104f204e0b8SIan Munsie start = local_clock(); 105f204e0b8SIan Munsie PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 106f204e0b8SIan Munsie while ((PSL_CNTL & CXL_PSL_SCNTL_An_Ps_MASK) 107f204e0b8SIan Munsie == CXL_PSL_SCNTL_An_Ps_Pending) { 108f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 109f204e0b8SIan Munsie dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n"); 110f204e0b8SIan Munsie return -EBUSY; 111f204e0b8SIan Munsie } 112f204e0b8SIan Munsie dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 113f204e0b8SIan Munsie pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%.16llx PSL_DSISR: 0x%.16llx\n", PSL_CNTL, dsisr); 114f204e0b8SIan Munsie if (dsisr & CXL_PSL_DSISR_TRANS) { 115f204e0b8SIan Munsie dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); 116f204e0b8SIan Munsie dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%.16llx, DAR: 0x%.16llx\n", dsisr, dar); 117f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE); 118f204e0b8SIan Munsie } else if (dsisr) { 119f204e0b8SIan Munsie dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%.16llx\n", dsisr); 120f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A); 121f204e0b8SIan Munsie } else { 122f204e0b8SIan Munsie cpu_relax(); 123f204e0b8SIan Munsie } 124f204e0b8SIan Munsie PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 125f204e0b8SIan Munsie }; 126f204e0b8SIan Munsie end = local_clock(); 127f204e0b8SIan Munsie pr_devel("PSL purged in %lld ns\n", end - start); 128f204e0b8SIan Munsie 129f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SCNTL_An, 130f204e0b8SIan Munsie PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc); 131f204e0b8SIan Munsie return 0; 132f204e0b8SIan Munsie } 133f204e0b8SIan Munsie 134f204e0b8SIan Munsie static int spa_max_procs(int spa_size) 135f204e0b8SIan Munsie { 136f204e0b8SIan Munsie /* 137f204e0b8SIan Munsie * From the CAIA: 138f204e0b8SIan Munsie * end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255 139f204e0b8SIan Munsie * Most of that junk is really just an overly-complicated way of saying 140f204e0b8SIan Munsie * the last 256 bytes are __aligned(128), so it's really: 141f204e0b8SIan Munsie * end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255 142f204e0b8SIan Munsie * and 143f204e0b8SIan Munsie * end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1 144f204e0b8SIan Munsie * so 145f204e0b8SIan Munsie * sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256 146f204e0b8SIan Munsie * Ignore the alignment (which is safe in this case as long as we are 147f204e0b8SIan Munsie * careful with our rounding) and solve for n: 148f204e0b8SIan Munsie */ 149f204e0b8SIan Munsie return ((spa_size / 8) - 96) / 17; 150f204e0b8SIan Munsie } 151f204e0b8SIan Munsie 152f204e0b8SIan Munsie static int alloc_spa(struct cxl_afu *afu) 153f204e0b8SIan Munsie { 154f204e0b8SIan Munsie u64 spap; 155f204e0b8SIan Munsie 156f204e0b8SIan Munsie /* Work out how many pages to allocate */ 157f204e0b8SIan Munsie afu->spa_order = 0; 158f204e0b8SIan Munsie do { 159f204e0b8SIan Munsie afu->spa_order++; 160f204e0b8SIan Munsie afu->spa_size = (1 << afu->spa_order) * PAGE_SIZE; 161f204e0b8SIan Munsie afu->spa_max_procs = spa_max_procs(afu->spa_size); 162f204e0b8SIan Munsie } while (afu->spa_max_procs < afu->num_procs); 163f204e0b8SIan Munsie 164f204e0b8SIan Munsie WARN_ON(afu->spa_size > 0x100000); /* Max size supported by the hardware */ 165f204e0b8SIan Munsie 166f204e0b8SIan Munsie if (!(afu->spa = (struct cxl_process_element *) 167f204e0b8SIan Munsie __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->spa_order))) { 168f204e0b8SIan Munsie pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n"); 169f204e0b8SIan Munsie return -ENOMEM; 170f204e0b8SIan Munsie } 171f204e0b8SIan Munsie pr_devel("spa pages: %i afu->spa_max_procs: %i afu->num_procs: %i\n", 172f204e0b8SIan Munsie 1<<afu->spa_order, afu->spa_max_procs, afu->num_procs); 173f204e0b8SIan Munsie 174f204e0b8SIan Munsie afu->sw_command_status = (__be64 *)((char *)afu->spa + 175f204e0b8SIan Munsie ((afu->spa_max_procs + 3) * 128)); 176f204e0b8SIan Munsie 177f204e0b8SIan Munsie spap = virt_to_phys(afu->spa) & CXL_PSL_SPAP_Addr; 178f204e0b8SIan Munsie spap |= ((afu->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size; 179f204e0b8SIan Munsie spap |= CXL_PSL_SPAP_V; 180f204e0b8SIan Munsie pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n", afu->spa, afu->spa_max_procs, afu->sw_command_status, spap); 181f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap); 182f204e0b8SIan Munsie 183f204e0b8SIan Munsie return 0; 184f204e0b8SIan Munsie } 185f204e0b8SIan Munsie 186f204e0b8SIan Munsie static void release_spa(struct cxl_afu *afu) 187f204e0b8SIan Munsie { 188db7933f3SIan Munsie cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); 189f204e0b8SIan Munsie free_pages((unsigned long) afu->spa, afu->spa_order); 190f204e0b8SIan Munsie } 191f204e0b8SIan Munsie 192f204e0b8SIan Munsie int cxl_tlb_slb_invalidate(struct cxl *adapter) 193f204e0b8SIan Munsie { 194f204e0b8SIan Munsie unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 195f204e0b8SIan Munsie 196f204e0b8SIan Munsie pr_devel("CXL adapter wide TLBIA & SLBIA\n"); 197f204e0b8SIan Munsie 198f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A); 199f204e0b8SIan Munsie 200f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL); 201f204e0b8SIan Munsie while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) { 202f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 203f204e0b8SIan Munsie dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n"); 204f204e0b8SIan Munsie return -EBUSY; 205f204e0b8SIan Munsie } 206f204e0b8SIan Munsie cpu_relax(); 207f204e0b8SIan Munsie } 208f204e0b8SIan Munsie 209f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL); 210f204e0b8SIan Munsie while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) { 211f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 212f204e0b8SIan Munsie dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n"); 213f204e0b8SIan Munsie return -EBUSY; 214f204e0b8SIan Munsie } 215f204e0b8SIan Munsie cpu_relax(); 216f204e0b8SIan Munsie } 217f204e0b8SIan Munsie return 0; 218f204e0b8SIan Munsie } 219f204e0b8SIan Munsie 220f204e0b8SIan Munsie int cxl_afu_slbia(struct cxl_afu *afu) 221f204e0b8SIan Munsie { 222f204e0b8SIan Munsie unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 223f204e0b8SIan Munsie 224f204e0b8SIan Munsie pr_devel("cxl_afu_slbia issuing SLBIA command\n"); 225f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_SLBIA_An, CXL_TLB_SLB_IQ_ALL); 226f204e0b8SIan Munsie while (cxl_p2n_read(afu, CXL_SLBIA_An) & CXL_TLB_SLB_P) { 227f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 228f204e0b8SIan Munsie dev_warn(&afu->dev, "WARNING: CXL AFU SLBIA timed out!\n"); 229f204e0b8SIan Munsie return -EBUSY; 230f204e0b8SIan Munsie } 231f204e0b8SIan Munsie cpu_relax(); 232f204e0b8SIan Munsie } 233f204e0b8SIan Munsie return 0; 234f204e0b8SIan Munsie } 235f204e0b8SIan Munsie 236f204e0b8SIan Munsie static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1) 237f204e0b8SIan Munsie { 238f204e0b8SIan Munsie int rc; 239f204e0b8SIan Munsie 240f204e0b8SIan Munsie /* 1. Disable SSTP by writing 0 to SSTP1[V] */ 241f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_SSTP1_An, 0); 242f204e0b8SIan Munsie 243f204e0b8SIan Munsie /* 2. Invalidate all SLB entries */ 244f204e0b8SIan Munsie if ((rc = cxl_afu_slbia(afu))) 245f204e0b8SIan Munsie return rc; 246f204e0b8SIan Munsie 247f204e0b8SIan Munsie /* 3. Set SSTP0_An */ 248f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_SSTP0_An, sstp0); 249f204e0b8SIan Munsie 250f204e0b8SIan Munsie /* 4. Set SSTP1_An */ 251f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_SSTP1_An, sstp1); 252f204e0b8SIan Munsie 253f204e0b8SIan Munsie return 0; 254f204e0b8SIan Munsie } 255f204e0b8SIan Munsie 256f204e0b8SIan Munsie /* Using per slice version may improve performance here. (ie. SLBIA_An) */ 257f204e0b8SIan Munsie static void slb_invalid(struct cxl_context *ctx) 258f204e0b8SIan Munsie { 259f204e0b8SIan Munsie struct cxl *adapter = ctx->afu->adapter; 260f204e0b8SIan Munsie u64 slbia; 261f204e0b8SIan Munsie 262f204e0b8SIan Munsie WARN_ON(!mutex_is_locked(&ctx->afu->spa_mutex)); 263f204e0b8SIan Munsie 264f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_LBISEL, 265f204e0b8SIan Munsie ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) | 266f204e0b8SIan Munsie be32_to_cpu(ctx->elem->lpid)); 267f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID); 268f204e0b8SIan Munsie 269f204e0b8SIan Munsie while (1) { 270f204e0b8SIan Munsie slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA); 271f204e0b8SIan Munsie if (!(slbia & CXL_TLB_SLB_P)) 272f204e0b8SIan Munsie break; 273f204e0b8SIan Munsie cpu_relax(); 274f204e0b8SIan Munsie } 275f204e0b8SIan Munsie } 276f204e0b8SIan Munsie 277f204e0b8SIan Munsie static int do_process_element_cmd(struct cxl_context *ctx, 278f204e0b8SIan Munsie u64 cmd, u64 pe_state) 279f204e0b8SIan Munsie { 280f204e0b8SIan Munsie u64 state; 281a98e6e9fSIan Munsie unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 282f204e0b8SIan Munsie 283f204e0b8SIan Munsie WARN_ON(!ctx->afu->enabled); 284f204e0b8SIan Munsie 285f204e0b8SIan Munsie ctx->elem->software_state = cpu_to_be32(pe_state); 286f204e0b8SIan Munsie smp_wmb(); 287f204e0b8SIan Munsie *(ctx->afu->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe); 288f204e0b8SIan Munsie smp_mb(); 289f204e0b8SIan Munsie cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe); 290f204e0b8SIan Munsie while (1) { 291a98e6e9fSIan Munsie if (time_after_eq(jiffies, timeout)) { 292a98e6e9fSIan Munsie dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n"); 293a98e6e9fSIan Munsie return -EBUSY; 294a98e6e9fSIan Munsie } 295f204e0b8SIan Munsie state = be64_to_cpup(ctx->afu->sw_command_status); 296f204e0b8SIan Munsie if (state == ~0ULL) { 297f204e0b8SIan Munsie pr_err("cxl: Error adding process element to AFU\n"); 298f204e0b8SIan Munsie return -1; 299f204e0b8SIan Munsie } 300f204e0b8SIan Munsie if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK | CXL_SPA_SW_LINK_MASK)) == 301f204e0b8SIan Munsie (cmd | (cmd >> 16) | ctx->pe)) 302f204e0b8SIan Munsie break; 303f204e0b8SIan Munsie /* 304f204e0b8SIan Munsie * The command won't finish in the PSL if there are 305f204e0b8SIan Munsie * outstanding DSIs. Hence we need to yield here in 306f204e0b8SIan Munsie * case there are outstanding DSIs that we need to 307f204e0b8SIan Munsie * service. Tuning possiblity: we could wait for a 308f204e0b8SIan Munsie * while before sched 309f204e0b8SIan Munsie */ 310f204e0b8SIan Munsie schedule(); 311f204e0b8SIan Munsie 312f204e0b8SIan Munsie } 313f204e0b8SIan Munsie return 0; 314f204e0b8SIan Munsie } 315f204e0b8SIan Munsie 316f204e0b8SIan Munsie static int add_process_element(struct cxl_context *ctx) 317f204e0b8SIan Munsie { 318f204e0b8SIan Munsie int rc = 0; 319f204e0b8SIan Munsie 320f204e0b8SIan Munsie mutex_lock(&ctx->afu->spa_mutex); 321f204e0b8SIan Munsie pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe); 322f204e0b8SIan Munsie if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V))) 323f204e0b8SIan Munsie ctx->pe_inserted = true; 324f204e0b8SIan Munsie pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe); 325f204e0b8SIan Munsie mutex_unlock(&ctx->afu->spa_mutex); 326f204e0b8SIan Munsie return rc; 327f204e0b8SIan Munsie } 328f204e0b8SIan Munsie 329f204e0b8SIan Munsie static int terminate_process_element(struct cxl_context *ctx) 330f204e0b8SIan Munsie { 331f204e0b8SIan Munsie int rc = 0; 332f204e0b8SIan Munsie 333f204e0b8SIan Munsie /* fast path terminate if it's already invalid */ 334f204e0b8SIan Munsie if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V))) 335f204e0b8SIan Munsie return rc; 336f204e0b8SIan Munsie 337f204e0b8SIan Munsie mutex_lock(&ctx->afu->spa_mutex); 338f204e0b8SIan Munsie pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe); 339f204e0b8SIan Munsie rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE, 340f204e0b8SIan Munsie CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T); 341f204e0b8SIan Munsie ctx->elem->software_state = 0; /* Remove Valid bit */ 342f204e0b8SIan Munsie pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe); 343f204e0b8SIan Munsie mutex_unlock(&ctx->afu->spa_mutex); 344f204e0b8SIan Munsie return rc; 345f204e0b8SIan Munsie } 346f204e0b8SIan Munsie 347f204e0b8SIan Munsie static int remove_process_element(struct cxl_context *ctx) 348f204e0b8SIan Munsie { 349f204e0b8SIan Munsie int rc = 0; 350f204e0b8SIan Munsie 351f204e0b8SIan Munsie mutex_lock(&ctx->afu->spa_mutex); 352f204e0b8SIan Munsie pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe); 353f204e0b8SIan Munsie if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0))) 354f204e0b8SIan Munsie ctx->pe_inserted = false; 355f204e0b8SIan Munsie slb_invalid(ctx); 356f204e0b8SIan Munsie pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe); 357f204e0b8SIan Munsie mutex_unlock(&ctx->afu->spa_mutex); 358f204e0b8SIan Munsie 359f204e0b8SIan Munsie return rc; 360f204e0b8SIan Munsie } 361f204e0b8SIan Munsie 362f204e0b8SIan Munsie 363f204e0b8SIan Munsie static void assign_psn_space(struct cxl_context *ctx) 364f204e0b8SIan Munsie { 365f204e0b8SIan Munsie if (!ctx->afu->pp_size || ctx->master) { 366f204e0b8SIan Munsie ctx->psn_phys = ctx->afu->psn_phys; 367f204e0b8SIan Munsie ctx->psn_size = ctx->afu->adapter->ps_size; 368f204e0b8SIan Munsie } else { 369f204e0b8SIan Munsie ctx->psn_phys = ctx->afu->psn_phys + 370f204e0b8SIan Munsie (ctx->afu->pp_offset + ctx->afu->pp_size * ctx->pe); 371f204e0b8SIan Munsie ctx->psn_size = ctx->afu->pp_size; 372f204e0b8SIan Munsie } 373f204e0b8SIan Munsie } 374f204e0b8SIan Munsie 375f204e0b8SIan Munsie static int activate_afu_directed(struct cxl_afu *afu) 376f204e0b8SIan Munsie { 377f204e0b8SIan Munsie int rc; 378f204e0b8SIan Munsie 379f204e0b8SIan Munsie dev_info(&afu->dev, "Activating AFU directed mode\n"); 380f204e0b8SIan Munsie 381f204e0b8SIan Munsie if (alloc_spa(afu)) 382f204e0b8SIan Munsie return -ENOMEM; 383f204e0b8SIan Munsie 384f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU); 385f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); 386f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L); 387f204e0b8SIan Munsie 388f204e0b8SIan Munsie afu->current_mode = CXL_MODE_DIRECTED; 389f204e0b8SIan Munsie afu->num_procs = afu->max_procs_virtualised; 390f204e0b8SIan Munsie 391f204e0b8SIan Munsie if ((rc = cxl_chardev_m_afu_add(afu))) 392f204e0b8SIan Munsie return rc; 393f204e0b8SIan Munsie 394f204e0b8SIan Munsie if ((rc = cxl_sysfs_afu_m_add(afu))) 395f204e0b8SIan Munsie goto err; 396f204e0b8SIan Munsie 397f204e0b8SIan Munsie if ((rc = cxl_chardev_s_afu_add(afu))) 398f204e0b8SIan Munsie goto err1; 399f204e0b8SIan Munsie 400f204e0b8SIan Munsie return 0; 401f204e0b8SIan Munsie err1: 402f204e0b8SIan Munsie cxl_sysfs_afu_m_remove(afu); 403f204e0b8SIan Munsie err: 404f204e0b8SIan Munsie cxl_chardev_afu_remove(afu); 405f204e0b8SIan Munsie return rc; 406f204e0b8SIan Munsie } 407f204e0b8SIan Munsie 408f204e0b8SIan Munsie #ifdef CONFIG_CPU_LITTLE_ENDIAN 409f204e0b8SIan Munsie #define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE) 410f204e0b8SIan Munsie #else 411f204e0b8SIan Munsie #define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE)) 412f204e0b8SIan Munsie #endif 413f204e0b8SIan Munsie 414f204e0b8SIan Munsie static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr) 415f204e0b8SIan Munsie { 416f204e0b8SIan Munsie u64 sr; 417f204e0b8SIan Munsie int r, result; 418f204e0b8SIan Munsie 419f204e0b8SIan Munsie assign_psn_space(ctx); 420f204e0b8SIan Munsie 421f204e0b8SIan Munsie ctx->elem->ctxtime = 0; /* disable */ 422f204e0b8SIan Munsie ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID)); 423f204e0b8SIan Munsie ctx->elem->haurp = 0; /* disable */ 424f204e0b8SIan Munsie ctx->elem->sdr = cpu_to_be64(mfspr(SPRN_SDR1)); 425f204e0b8SIan Munsie 4265100a9d6SIan Munsie sr = 0; 427f204e0b8SIan Munsie if (ctx->master) 428f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_MP; 429f204e0b8SIan Munsie if (mfspr(SPRN_LPCR) & LPCR_TC) 430f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_TC; 431f204e0b8SIan Munsie /* HV=0, PR=1, R=1 for userspace 432f204e0b8SIan Munsie * For kernel contexts: this would need to change 433f204e0b8SIan Munsie */ 434f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R; 435f204e0b8SIan Munsie set_endian(sr); 436f204e0b8SIan Munsie sr &= ~(CXL_PSL_SR_An_HV); 437f204e0b8SIan Munsie if (!test_tsk_thread_flag(current, TIF_32BIT)) 438f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_SF; 439f204e0b8SIan Munsie ctx->elem->common.pid = cpu_to_be32(current->pid); 440f204e0b8SIan Munsie ctx->elem->common.tid = 0; 441f204e0b8SIan Munsie ctx->elem->sr = cpu_to_be64(sr); 442f204e0b8SIan Munsie 443f204e0b8SIan Munsie ctx->elem->common.csrp = 0; /* disable */ 444f204e0b8SIan Munsie ctx->elem->common.aurp0 = 0; /* disable */ 445f204e0b8SIan Munsie ctx->elem->common.aurp1 = 0; /* disable */ 446f204e0b8SIan Munsie 447f204e0b8SIan Munsie cxl_prefault(ctx, wed); 448f204e0b8SIan Munsie 449f204e0b8SIan Munsie ctx->elem->common.sstp0 = cpu_to_be64(ctx->sstp0); 450f204e0b8SIan Munsie ctx->elem->common.sstp1 = cpu_to_be64(ctx->sstp1); 451f204e0b8SIan Munsie 452f204e0b8SIan Munsie for (r = 0; r < CXL_IRQ_RANGES; r++) { 453f204e0b8SIan Munsie ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]); 454f204e0b8SIan Munsie ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]); 455f204e0b8SIan Munsie } 456f204e0b8SIan Munsie 457f204e0b8SIan Munsie ctx->elem->common.amr = cpu_to_be64(amr); 458f204e0b8SIan Munsie ctx->elem->common.wed = cpu_to_be64(wed); 459f204e0b8SIan Munsie 460f204e0b8SIan Munsie /* first guy needs to enable */ 461f204e0b8SIan Munsie if ((result = afu_check_and_enable(ctx->afu))) 462f204e0b8SIan Munsie return result; 463f204e0b8SIan Munsie 464f204e0b8SIan Munsie add_process_element(ctx); 465f204e0b8SIan Munsie 466f204e0b8SIan Munsie return 0; 467f204e0b8SIan Munsie } 468f204e0b8SIan Munsie 469f204e0b8SIan Munsie static int deactivate_afu_directed(struct cxl_afu *afu) 470f204e0b8SIan Munsie { 471f204e0b8SIan Munsie dev_info(&afu->dev, "Deactivating AFU directed mode\n"); 472f204e0b8SIan Munsie 473f204e0b8SIan Munsie afu->current_mode = 0; 474f204e0b8SIan Munsie afu->num_procs = 0; 475f204e0b8SIan Munsie 476f204e0b8SIan Munsie cxl_sysfs_afu_m_remove(afu); 477f204e0b8SIan Munsie cxl_chardev_afu_remove(afu); 478f204e0b8SIan Munsie 479f204e0b8SIan Munsie cxl_afu_reset(afu); 480f204e0b8SIan Munsie cxl_afu_disable(afu); 481f204e0b8SIan Munsie cxl_psl_purge(afu); 482f204e0b8SIan Munsie 483f204e0b8SIan Munsie release_spa(afu); 484f204e0b8SIan Munsie 485f204e0b8SIan Munsie return 0; 486f204e0b8SIan Munsie } 487f204e0b8SIan Munsie 488f204e0b8SIan Munsie static int activate_dedicated_process(struct cxl_afu *afu) 489f204e0b8SIan Munsie { 490f204e0b8SIan Munsie dev_info(&afu->dev, "Activating dedicated process mode\n"); 491f204e0b8SIan Munsie 492f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process); 493f204e0b8SIan Munsie 494f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */ 495f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); /* disable */ 496f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); 497f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID)); 498f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_HAURP_An, 0); /* disable */ 499f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1)); 500f204e0b8SIan Munsie 501f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_CSRP_An, 0); /* disable */ 502f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_AURP0_An, 0); /* disable */ 503f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_AURP1_An, 0); /* disable */ 504f204e0b8SIan Munsie 505f204e0b8SIan Munsie afu->current_mode = CXL_MODE_DEDICATED; 506f204e0b8SIan Munsie afu->num_procs = 1; 507f204e0b8SIan Munsie 508f204e0b8SIan Munsie return cxl_chardev_d_afu_add(afu); 509f204e0b8SIan Munsie } 510f204e0b8SIan Munsie 511f204e0b8SIan Munsie static int attach_dedicated(struct cxl_context *ctx, u64 wed, u64 amr) 512f204e0b8SIan Munsie { 513f204e0b8SIan Munsie struct cxl_afu *afu = ctx->afu; 514f204e0b8SIan Munsie u64 sr; 515f204e0b8SIan Munsie int rc; 516f204e0b8SIan Munsie 5175100a9d6SIan Munsie sr = 0; 518f204e0b8SIan Munsie set_endian(sr); 519f204e0b8SIan Munsie if (ctx->master) 520f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_MP; 521f204e0b8SIan Munsie if (mfspr(SPRN_LPCR) & LPCR_TC) 522f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_TC; 523f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R; 524f204e0b8SIan Munsie if (!test_tsk_thread_flag(current, TIF_32BIT)) 525f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_SF; 526f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_PID_TID_An, (u64)current->pid << 32); 527f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SR_An, sr); 528f204e0b8SIan Munsie 529f204e0b8SIan Munsie if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1))) 530f204e0b8SIan Munsie return rc; 531f204e0b8SIan Munsie 532f204e0b8SIan Munsie cxl_prefault(ctx, wed); 533f204e0b8SIan Munsie 534f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, 535f204e0b8SIan Munsie (((u64)ctx->irqs.offset[0] & 0xffff) << 48) | 536f204e0b8SIan Munsie (((u64)ctx->irqs.offset[1] & 0xffff) << 32) | 537f204e0b8SIan Munsie (((u64)ctx->irqs.offset[2] & 0xffff) << 16) | 538f204e0b8SIan Munsie ((u64)ctx->irqs.offset[3] & 0xffff)); 539f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64) 540f204e0b8SIan Munsie (((u64)ctx->irqs.range[0] & 0xffff) << 48) | 541f204e0b8SIan Munsie (((u64)ctx->irqs.range[1] & 0xffff) << 32) | 542f204e0b8SIan Munsie (((u64)ctx->irqs.range[2] & 0xffff) << 16) | 543f204e0b8SIan Munsie ((u64)ctx->irqs.range[3] & 0xffff)); 544f204e0b8SIan Munsie 545f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_AMR_An, amr); 546f204e0b8SIan Munsie 547f204e0b8SIan Munsie /* master only context for dedicated */ 548f204e0b8SIan Munsie assign_psn_space(ctx); 549f204e0b8SIan Munsie 550f204e0b8SIan Munsie if ((rc = cxl_afu_reset(afu))) 551f204e0b8SIan Munsie return rc; 552f204e0b8SIan Munsie 553f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_WED_An, wed); 554f204e0b8SIan Munsie 555f204e0b8SIan Munsie return afu_enable(afu); 556f204e0b8SIan Munsie } 557f204e0b8SIan Munsie 558f204e0b8SIan Munsie static int deactivate_dedicated_process(struct cxl_afu *afu) 559f204e0b8SIan Munsie { 560f204e0b8SIan Munsie dev_info(&afu->dev, "Deactivating dedicated process mode\n"); 561f204e0b8SIan Munsie 562f204e0b8SIan Munsie afu->current_mode = 0; 563f204e0b8SIan Munsie afu->num_procs = 0; 564f204e0b8SIan Munsie 565f204e0b8SIan Munsie cxl_chardev_afu_remove(afu); 566f204e0b8SIan Munsie 567f204e0b8SIan Munsie return 0; 568f204e0b8SIan Munsie } 569f204e0b8SIan Munsie 570f204e0b8SIan Munsie int _cxl_afu_deactivate_mode(struct cxl_afu *afu, int mode) 571f204e0b8SIan Munsie { 572f204e0b8SIan Munsie if (mode == CXL_MODE_DIRECTED) 573f204e0b8SIan Munsie return deactivate_afu_directed(afu); 574f204e0b8SIan Munsie if (mode == CXL_MODE_DEDICATED) 575f204e0b8SIan Munsie return deactivate_dedicated_process(afu); 576f204e0b8SIan Munsie return 0; 577f204e0b8SIan Munsie } 578f204e0b8SIan Munsie 579f204e0b8SIan Munsie int cxl_afu_deactivate_mode(struct cxl_afu *afu) 580f204e0b8SIan Munsie { 581f204e0b8SIan Munsie return _cxl_afu_deactivate_mode(afu, afu->current_mode); 582f204e0b8SIan Munsie } 583f204e0b8SIan Munsie 584f204e0b8SIan Munsie int cxl_afu_activate_mode(struct cxl_afu *afu, int mode) 585f204e0b8SIan Munsie { 586f204e0b8SIan Munsie if (!mode) 587f204e0b8SIan Munsie return 0; 588f204e0b8SIan Munsie if (!(mode & afu->modes_supported)) 589f204e0b8SIan Munsie return -EINVAL; 590f204e0b8SIan Munsie 591f204e0b8SIan Munsie if (mode == CXL_MODE_DIRECTED) 592f204e0b8SIan Munsie return activate_afu_directed(afu); 593f204e0b8SIan Munsie if (mode == CXL_MODE_DEDICATED) 594f204e0b8SIan Munsie return activate_dedicated_process(afu); 595f204e0b8SIan Munsie 596f204e0b8SIan Munsie return -EINVAL; 597f204e0b8SIan Munsie } 598f204e0b8SIan Munsie 599f204e0b8SIan Munsie int cxl_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr) 600f204e0b8SIan Munsie { 601f204e0b8SIan Munsie ctx->kernel = kernel; 602f204e0b8SIan Munsie if (ctx->afu->current_mode == CXL_MODE_DIRECTED) 603f204e0b8SIan Munsie return attach_afu_directed(ctx, wed, amr); 604f204e0b8SIan Munsie 605f204e0b8SIan Munsie if (ctx->afu->current_mode == CXL_MODE_DEDICATED) 606f204e0b8SIan Munsie return attach_dedicated(ctx, wed, amr); 607f204e0b8SIan Munsie 608f204e0b8SIan Munsie return -EINVAL; 609f204e0b8SIan Munsie } 610f204e0b8SIan Munsie 611f204e0b8SIan Munsie static inline int detach_process_native_dedicated(struct cxl_context *ctx) 612f204e0b8SIan Munsie { 613f204e0b8SIan Munsie cxl_afu_reset(ctx->afu); 614f204e0b8SIan Munsie cxl_afu_disable(ctx->afu); 615f204e0b8SIan Munsie cxl_psl_purge(ctx->afu); 616f204e0b8SIan Munsie return 0; 617f204e0b8SIan Munsie } 618f204e0b8SIan Munsie 619f204e0b8SIan Munsie static inline int detach_process_native_afu_directed(struct cxl_context *ctx) 620f204e0b8SIan Munsie { 621f204e0b8SIan Munsie if (!ctx->pe_inserted) 622f204e0b8SIan Munsie return 0; 623f204e0b8SIan Munsie if (terminate_process_element(ctx)) 624f204e0b8SIan Munsie return -1; 625f204e0b8SIan Munsie if (remove_process_element(ctx)) 626f204e0b8SIan Munsie return -1; 627f204e0b8SIan Munsie 628f204e0b8SIan Munsie return 0; 629f204e0b8SIan Munsie } 630f204e0b8SIan Munsie 631f204e0b8SIan Munsie int cxl_detach_process(struct cxl_context *ctx) 632f204e0b8SIan Munsie { 633f204e0b8SIan Munsie if (ctx->afu->current_mode == CXL_MODE_DEDICATED) 634f204e0b8SIan Munsie return detach_process_native_dedicated(ctx); 635f204e0b8SIan Munsie 636f204e0b8SIan Munsie return detach_process_native_afu_directed(ctx); 637f204e0b8SIan Munsie } 638f204e0b8SIan Munsie 639bc78b05bSIan Munsie int cxl_get_irq(struct cxl_afu *afu, struct cxl_irq_info *info) 640f204e0b8SIan Munsie { 641f204e0b8SIan Munsie u64 pidtid; 642f204e0b8SIan Munsie 643bc78b05bSIan Munsie info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 644bc78b05bSIan Munsie info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); 645bc78b05bSIan Munsie info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An); 646bc78b05bSIan Munsie pidtid = cxl_p2n_read(afu, CXL_PSL_PID_TID_An); 647f204e0b8SIan Munsie info->pid = pidtid >> 32; 648f204e0b8SIan Munsie info->tid = pidtid & 0xffffffff; 649bc78b05bSIan Munsie info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An); 650bc78b05bSIan Munsie info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An); 651f204e0b8SIan Munsie 652f204e0b8SIan Munsie return 0; 653f204e0b8SIan Munsie } 654f204e0b8SIan Munsie 655f204e0b8SIan Munsie static void recover_psl_err(struct cxl_afu *afu, u64 errstat) 656f204e0b8SIan Munsie { 657f204e0b8SIan Munsie u64 dsisr; 658f204e0b8SIan Munsie 659f204e0b8SIan Munsie pr_devel("RECOVERING FROM PSL ERROR... (0x%.16llx)\n", errstat); 660f204e0b8SIan Munsie 661f204e0b8SIan Munsie /* Clear PSL_DSISR[PE] */ 662f204e0b8SIan Munsie dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 663f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE); 664f204e0b8SIan Munsie 665f204e0b8SIan Munsie /* Write 1s to clear error status bits */ 666f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat); 667f204e0b8SIan Munsie } 668f204e0b8SIan Munsie 669f204e0b8SIan Munsie int cxl_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask) 670f204e0b8SIan Munsie { 671f204e0b8SIan Munsie if (tfc) 672f204e0b8SIan Munsie cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc); 673f204e0b8SIan Munsie if (psl_reset_mask) 674f204e0b8SIan Munsie recover_psl_err(ctx->afu, psl_reset_mask); 675f204e0b8SIan Munsie 676f204e0b8SIan Munsie return 0; 677f204e0b8SIan Munsie } 678f204e0b8SIan Munsie 679f204e0b8SIan Munsie int cxl_check_error(struct cxl_afu *afu) 680f204e0b8SIan Munsie { 681f204e0b8SIan Munsie return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL); 682f204e0b8SIan Munsie } 683