1f204e0b8SIan Munsie /* 2f204e0b8SIan Munsie * Copyright 2014 IBM Corp. 3f204e0b8SIan Munsie * 4f204e0b8SIan Munsie * This program is free software; you can redistribute it and/or 5f204e0b8SIan Munsie * modify it under the terms of the GNU General Public License 6f204e0b8SIan Munsie * as published by the Free Software Foundation; either version 7f204e0b8SIan Munsie * 2 of the License, or (at your option) any later version. 8f204e0b8SIan Munsie */ 9f204e0b8SIan Munsie 10f204e0b8SIan Munsie #include <linux/spinlock.h> 11f204e0b8SIan Munsie #include <linux/sched.h> 12f204e0b8SIan Munsie #include <linux/slab.h> 13f204e0b8SIan Munsie #include <linux/sched.h> 14f204e0b8SIan Munsie #include <linux/mutex.h> 15f204e0b8SIan Munsie #include <linux/mm.h> 16f204e0b8SIan Munsie #include <linux/uaccess.h> 17f204e0b8SIan Munsie #include <asm/synch.h> 18f204e0b8SIan Munsie #include <misc/cxl.h> 19f204e0b8SIan Munsie 20f204e0b8SIan Munsie #include "cxl.h" 21f204e0b8SIan Munsie 22f204e0b8SIan Munsie static int afu_control(struct cxl_afu *afu, u64 command, 23f204e0b8SIan Munsie u64 result, u64 mask, bool enabled) 24f204e0b8SIan Munsie { 25f204e0b8SIan Munsie u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 26f204e0b8SIan Munsie unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 27f204e0b8SIan Munsie 28f204e0b8SIan Munsie spin_lock(&afu->afu_cntl_lock); 29f204e0b8SIan Munsie pr_devel("AFU command starting: %llx\n", command); 30f204e0b8SIan Munsie 31f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl | command); 32f204e0b8SIan Munsie 33f204e0b8SIan Munsie AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 34f204e0b8SIan Munsie while ((AFU_Cntl & mask) != result) { 35f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 36f204e0b8SIan Munsie dev_warn(&afu->dev, "WARNING: AFU control timed out!\n"); 37f204e0b8SIan Munsie spin_unlock(&afu->afu_cntl_lock); 38f204e0b8SIan Munsie return -EBUSY; 39f204e0b8SIan Munsie } 40f204e0b8SIan Munsie pr_devel_ratelimited("AFU control... (0x%.16llx)\n", 41f204e0b8SIan Munsie AFU_Cntl | command); 42f204e0b8SIan Munsie cpu_relax(); 43f204e0b8SIan Munsie AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 44f204e0b8SIan Munsie }; 45f204e0b8SIan Munsie pr_devel("AFU command complete: %llx\n", command); 46f204e0b8SIan Munsie afu->enabled = enabled; 47f204e0b8SIan Munsie spin_unlock(&afu->afu_cntl_lock); 48f204e0b8SIan Munsie 49f204e0b8SIan Munsie return 0; 50f204e0b8SIan Munsie } 51f204e0b8SIan Munsie 52f204e0b8SIan Munsie static int afu_enable(struct cxl_afu *afu) 53f204e0b8SIan Munsie { 54f204e0b8SIan Munsie pr_devel("AFU enable request\n"); 55f204e0b8SIan Munsie 56f204e0b8SIan Munsie return afu_control(afu, CXL_AFU_Cntl_An_E, 57f204e0b8SIan Munsie CXL_AFU_Cntl_An_ES_Enabled, 58f204e0b8SIan Munsie CXL_AFU_Cntl_An_ES_MASK, true); 59f204e0b8SIan Munsie } 60f204e0b8SIan Munsie 61f204e0b8SIan Munsie int cxl_afu_disable(struct cxl_afu *afu) 62f204e0b8SIan Munsie { 63f204e0b8SIan Munsie pr_devel("AFU disable request\n"); 64f204e0b8SIan Munsie 65f204e0b8SIan Munsie return afu_control(afu, 0, CXL_AFU_Cntl_An_ES_Disabled, 66f204e0b8SIan Munsie CXL_AFU_Cntl_An_ES_MASK, false); 67f204e0b8SIan Munsie } 68f204e0b8SIan Munsie 69f204e0b8SIan Munsie /* This will disable as well as reset */ 70f204e0b8SIan Munsie int cxl_afu_reset(struct cxl_afu *afu) 71f204e0b8SIan Munsie { 72f204e0b8SIan Munsie pr_devel("AFU reset request\n"); 73f204e0b8SIan Munsie 74f204e0b8SIan Munsie return afu_control(afu, CXL_AFU_Cntl_An_RA, 75f204e0b8SIan Munsie CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled, 76f204e0b8SIan Munsie CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK, 77f204e0b8SIan Munsie false); 78f204e0b8SIan Munsie } 79f204e0b8SIan Munsie 80f204e0b8SIan Munsie static int afu_check_and_enable(struct cxl_afu *afu) 81f204e0b8SIan Munsie { 82f204e0b8SIan Munsie if (afu->enabled) 83f204e0b8SIan Munsie return 0; 84f204e0b8SIan Munsie return afu_enable(afu); 85f204e0b8SIan Munsie } 86f204e0b8SIan Munsie 87f204e0b8SIan Munsie int cxl_psl_purge(struct cxl_afu *afu) 88f204e0b8SIan Munsie { 89f204e0b8SIan Munsie u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 90f204e0b8SIan Munsie u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); 91f204e0b8SIan Munsie u64 dsisr, dar; 92f204e0b8SIan Munsie u64 start, end; 93f204e0b8SIan Munsie unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 94f204e0b8SIan Munsie 95f204e0b8SIan Munsie pr_devel("PSL purge request\n"); 96f204e0b8SIan Munsie 97f204e0b8SIan Munsie if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) { 98f204e0b8SIan Munsie WARN(1, "psl_purge request while AFU not disabled!\n"); 99f204e0b8SIan Munsie cxl_afu_disable(afu); 100f204e0b8SIan Munsie } 101f204e0b8SIan Munsie 102f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SCNTL_An, 103f204e0b8SIan Munsie PSL_CNTL | CXL_PSL_SCNTL_An_Pc); 104f204e0b8SIan Munsie start = local_clock(); 105f204e0b8SIan Munsie PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 106f204e0b8SIan Munsie while ((PSL_CNTL & CXL_PSL_SCNTL_An_Ps_MASK) 107f204e0b8SIan Munsie == CXL_PSL_SCNTL_An_Ps_Pending) { 108f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 109f204e0b8SIan Munsie dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n"); 110f204e0b8SIan Munsie return -EBUSY; 111f204e0b8SIan Munsie } 112f204e0b8SIan Munsie dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 113f204e0b8SIan Munsie pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%.16llx PSL_DSISR: 0x%.16llx\n", PSL_CNTL, dsisr); 114f204e0b8SIan Munsie if (dsisr & CXL_PSL_DSISR_TRANS) { 115f204e0b8SIan Munsie dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); 116f204e0b8SIan Munsie dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%.16llx, DAR: 0x%.16llx\n", dsisr, dar); 117f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE); 118f204e0b8SIan Munsie } else if (dsisr) { 119f204e0b8SIan Munsie dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%.16llx\n", dsisr); 120f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A); 121f204e0b8SIan Munsie } else { 122f204e0b8SIan Munsie cpu_relax(); 123f204e0b8SIan Munsie } 124f204e0b8SIan Munsie PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); 125f204e0b8SIan Munsie }; 126f204e0b8SIan Munsie end = local_clock(); 127f204e0b8SIan Munsie pr_devel("PSL purged in %lld ns\n", end - start); 128f204e0b8SIan Munsie 129f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SCNTL_An, 130f204e0b8SIan Munsie PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc); 131f204e0b8SIan Munsie return 0; 132f204e0b8SIan Munsie } 133f204e0b8SIan Munsie 134f204e0b8SIan Munsie static int spa_max_procs(int spa_size) 135f204e0b8SIan Munsie { 136f204e0b8SIan Munsie /* 137f204e0b8SIan Munsie * From the CAIA: 138f204e0b8SIan Munsie * end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255 139f204e0b8SIan Munsie * Most of that junk is really just an overly-complicated way of saying 140f204e0b8SIan Munsie * the last 256 bytes are __aligned(128), so it's really: 141f204e0b8SIan Munsie * end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255 142f204e0b8SIan Munsie * and 143f204e0b8SIan Munsie * end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1 144f204e0b8SIan Munsie * so 145f204e0b8SIan Munsie * sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256 146f204e0b8SIan Munsie * Ignore the alignment (which is safe in this case as long as we are 147f204e0b8SIan Munsie * careful with our rounding) and solve for n: 148f204e0b8SIan Munsie */ 149f204e0b8SIan Munsie return ((spa_size / 8) - 96) / 17; 150f204e0b8SIan Munsie } 151f204e0b8SIan Munsie 152f204e0b8SIan Munsie static int alloc_spa(struct cxl_afu *afu) 153f204e0b8SIan Munsie { 154f204e0b8SIan Munsie u64 spap; 155f204e0b8SIan Munsie 156f204e0b8SIan Munsie /* Work out how many pages to allocate */ 157f204e0b8SIan Munsie afu->spa_order = 0; 158f204e0b8SIan Munsie do { 159f204e0b8SIan Munsie afu->spa_order++; 160f204e0b8SIan Munsie afu->spa_size = (1 << afu->spa_order) * PAGE_SIZE; 161f204e0b8SIan Munsie afu->spa_max_procs = spa_max_procs(afu->spa_size); 162f204e0b8SIan Munsie } while (afu->spa_max_procs < afu->num_procs); 163f204e0b8SIan Munsie 164f204e0b8SIan Munsie WARN_ON(afu->spa_size > 0x100000); /* Max size supported by the hardware */ 165f204e0b8SIan Munsie 166f204e0b8SIan Munsie if (!(afu->spa = (struct cxl_process_element *) 167f204e0b8SIan Munsie __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->spa_order))) { 168f204e0b8SIan Munsie pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n"); 169f204e0b8SIan Munsie return -ENOMEM; 170f204e0b8SIan Munsie } 171f204e0b8SIan Munsie pr_devel("spa pages: %i afu->spa_max_procs: %i afu->num_procs: %i\n", 172f204e0b8SIan Munsie 1<<afu->spa_order, afu->spa_max_procs, afu->num_procs); 173f204e0b8SIan Munsie 174f204e0b8SIan Munsie afu->sw_command_status = (__be64 *)((char *)afu->spa + 175f204e0b8SIan Munsie ((afu->spa_max_procs + 3) * 128)); 176f204e0b8SIan Munsie 177f204e0b8SIan Munsie spap = virt_to_phys(afu->spa) & CXL_PSL_SPAP_Addr; 178f204e0b8SIan Munsie spap |= ((afu->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size; 179f204e0b8SIan Munsie spap |= CXL_PSL_SPAP_V; 180f204e0b8SIan Munsie pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n", afu->spa, afu->spa_max_procs, afu->sw_command_status, spap); 181f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap); 182f204e0b8SIan Munsie 183f204e0b8SIan Munsie return 0; 184f204e0b8SIan Munsie } 185f204e0b8SIan Munsie 186f204e0b8SIan Munsie static void release_spa(struct cxl_afu *afu) 187f204e0b8SIan Munsie { 188f204e0b8SIan Munsie free_pages((unsigned long) afu->spa, afu->spa_order); 189f204e0b8SIan Munsie } 190f204e0b8SIan Munsie 191f204e0b8SIan Munsie int cxl_tlb_slb_invalidate(struct cxl *adapter) 192f204e0b8SIan Munsie { 193f204e0b8SIan Munsie unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 194f204e0b8SIan Munsie 195f204e0b8SIan Munsie pr_devel("CXL adapter wide TLBIA & SLBIA\n"); 196f204e0b8SIan Munsie 197f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A); 198f204e0b8SIan Munsie 199f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL); 200f204e0b8SIan Munsie while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) { 201f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 202f204e0b8SIan Munsie dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n"); 203f204e0b8SIan Munsie return -EBUSY; 204f204e0b8SIan Munsie } 205f204e0b8SIan Munsie cpu_relax(); 206f204e0b8SIan Munsie } 207f204e0b8SIan Munsie 208f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL); 209f204e0b8SIan Munsie while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) { 210f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 211f204e0b8SIan Munsie dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n"); 212f204e0b8SIan Munsie return -EBUSY; 213f204e0b8SIan Munsie } 214f204e0b8SIan Munsie cpu_relax(); 215f204e0b8SIan Munsie } 216f204e0b8SIan Munsie return 0; 217f204e0b8SIan Munsie } 218f204e0b8SIan Munsie 219f204e0b8SIan Munsie int cxl_afu_slbia(struct cxl_afu *afu) 220f204e0b8SIan Munsie { 221f204e0b8SIan Munsie unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT); 222f204e0b8SIan Munsie 223f204e0b8SIan Munsie pr_devel("cxl_afu_slbia issuing SLBIA command\n"); 224f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_SLBIA_An, CXL_TLB_SLB_IQ_ALL); 225f204e0b8SIan Munsie while (cxl_p2n_read(afu, CXL_SLBIA_An) & CXL_TLB_SLB_P) { 226f204e0b8SIan Munsie if (time_after_eq(jiffies, timeout)) { 227f204e0b8SIan Munsie dev_warn(&afu->dev, "WARNING: CXL AFU SLBIA timed out!\n"); 228f204e0b8SIan Munsie return -EBUSY; 229f204e0b8SIan Munsie } 230f204e0b8SIan Munsie cpu_relax(); 231f204e0b8SIan Munsie } 232f204e0b8SIan Munsie return 0; 233f204e0b8SIan Munsie } 234f204e0b8SIan Munsie 235f204e0b8SIan Munsie static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1) 236f204e0b8SIan Munsie { 237f204e0b8SIan Munsie int rc; 238f204e0b8SIan Munsie 239f204e0b8SIan Munsie /* 1. Disable SSTP by writing 0 to SSTP1[V] */ 240f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_SSTP1_An, 0); 241f204e0b8SIan Munsie 242f204e0b8SIan Munsie /* 2. Invalidate all SLB entries */ 243f204e0b8SIan Munsie if ((rc = cxl_afu_slbia(afu))) 244f204e0b8SIan Munsie return rc; 245f204e0b8SIan Munsie 246f204e0b8SIan Munsie /* 3. Set SSTP0_An */ 247f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_SSTP0_An, sstp0); 248f204e0b8SIan Munsie 249f204e0b8SIan Munsie /* 4. Set SSTP1_An */ 250f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_SSTP1_An, sstp1); 251f204e0b8SIan Munsie 252f204e0b8SIan Munsie return 0; 253f204e0b8SIan Munsie } 254f204e0b8SIan Munsie 255f204e0b8SIan Munsie /* Using per slice version may improve performance here. (ie. SLBIA_An) */ 256f204e0b8SIan Munsie static void slb_invalid(struct cxl_context *ctx) 257f204e0b8SIan Munsie { 258f204e0b8SIan Munsie struct cxl *adapter = ctx->afu->adapter; 259f204e0b8SIan Munsie u64 slbia; 260f204e0b8SIan Munsie 261f204e0b8SIan Munsie WARN_ON(!mutex_is_locked(&ctx->afu->spa_mutex)); 262f204e0b8SIan Munsie 263f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_LBISEL, 264f204e0b8SIan Munsie ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) | 265f204e0b8SIan Munsie be32_to_cpu(ctx->elem->lpid)); 266f204e0b8SIan Munsie cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID); 267f204e0b8SIan Munsie 268f204e0b8SIan Munsie while (1) { 269f204e0b8SIan Munsie slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA); 270f204e0b8SIan Munsie if (!(slbia & CXL_TLB_SLB_P)) 271f204e0b8SIan Munsie break; 272f204e0b8SIan Munsie cpu_relax(); 273f204e0b8SIan Munsie } 274f204e0b8SIan Munsie } 275f204e0b8SIan Munsie 276f204e0b8SIan Munsie static int do_process_element_cmd(struct cxl_context *ctx, 277f204e0b8SIan Munsie u64 cmd, u64 pe_state) 278f204e0b8SIan Munsie { 279f204e0b8SIan Munsie u64 state; 280f204e0b8SIan Munsie 281f204e0b8SIan Munsie WARN_ON(!ctx->afu->enabled); 282f204e0b8SIan Munsie 283f204e0b8SIan Munsie ctx->elem->software_state = cpu_to_be32(pe_state); 284f204e0b8SIan Munsie smp_wmb(); 285f204e0b8SIan Munsie *(ctx->afu->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe); 286f204e0b8SIan Munsie smp_mb(); 287f204e0b8SIan Munsie cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe); 288f204e0b8SIan Munsie while (1) { 289f204e0b8SIan Munsie state = be64_to_cpup(ctx->afu->sw_command_status); 290f204e0b8SIan Munsie if (state == ~0ULL) { 291f204e0b8SIan Munsie pr_err("cxl: Error adding process element to AFU\n"); 292f204e0b8SIan Munsie return -1; 293f204e0b8SIan Munsie } 294f204e0b8SIan Munsie if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK | CXL_SPA_SW_LINK_MASK)) == 295f204e0b8SIan Munsie (cmd | (cmd >> 16) | ctx->pe)) 296f204e0b8SIan Munsie break; 297f204e0b8SIan Munsie /* 298f204e0b8SIan Munsie * The command won't finish in the PSL if there are 299f204e0b8SIan Munsie * outstanding DSIs. Hence we need to yield here in 300f204e0b8SIan Munsie * case there are outstanding DSIs that we need to 301f204e0b8SIan Munsie * service. Tuning possiblity: we could wait for a 302f204e0b8SIan Munsie * while before sched 303f204e0b8SIan Munsie */ 304f204e0b8SIan Munsie schedule(); 305f204e0b8SIan Munsie 306f204e0b8SIan Munsie } 307f204e0b8SIan Munsie return 0; 308f204e0b8SIan Munsie } 309f204e0b8SIan Munsie 310f204e0b8SIan Munsie static int add_process_element(struct cxl_context *ctx) 311f204e0b8SIan Munsie { 312f204e0b8SIan Munsie int rc = 0; 313f204e0b8SIan Munsie 314f204e0b8SIan Munsie mutex_lock(&ctx->afu->spa_mutex); 315f204e0b8SIan Munsie pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe); 316f204e0b8SIan Munsie if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V))) 317f204e0b8SIan Munsie ctx->pe_inserted = true; 318f204e0b8SIan Munsie pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe); 319f204e0b8SIan Munsie mutex_unlock(&ctx->afu->spa_mutex); 320f204e0b8SIan Munsie return rc; 321f204e0b8SIan Munsie } 322f204e0b8SIan Munsie 323f204e0b8SIan Munsie static int terminate_process_element(struct cxl_context *ctx) 324f204e0b8SIan Munsie { 325f204e0b8SIan Munsie int rc = 0; 326f204e0b8SIan Munsie 327f204e0b8SIan Munsie /* fast path terminate if it's already invalid */ 328f204e0b8SIan Munsie if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V))) 329f204e0b8SIan Munsie return rc; 330f204e0b8SIan Munsie 331f204e0b8SIan Munsie mutex_lock(&ctx->afu->spa_mutex); 332f204e0b8SIan Munsie pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe); 333f204e0b8SIan Munsie rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE, 334f204e0b8SIan Munsie CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T); 335f204e0b8SIan Munsie ctx->elem->software_state = 0; /* Remove Valid bit */ 336f204e0b8SIan Munsie pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe); 337f204e0b8SIan Munsie mutex_unlock(&ctx->afu->spa_mutex); 338f204e0b8SIan Munsie return rc; 339f204e0b8SIan Munsie } 340f204e0b8SIan Munsie 341f204e0b8SIan Munsie static int remove_process_element(struct cxl_context *ctx) 342f204e0b8SIan Munsie { 343f204e0b8SIan Munsie int rc = 0; 344f204e0b8SIan Munsie 345f204e0b8SIan Munsie mutex_lock(&ctx->afu->spa_mutex); 346f204e0b8SIan Munsie pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe); 347f204e0b8SIan Munsie if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0))) 348f204e0b8SIan Munsie ctx->pe_inserted = false; 349f204e0b8SIan Munsie slb_invalid(ctx); 350f204e0b8SIan Munsie pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe); 351f204e0b8SIan Munsie mutex_unlock(&ctx->afu->spa_mutex); 352f204e0b8SIan Munsie 353f204e0b8SIan Munsie return rc; 354f204e0b8SIan Munsie } 355f204e0b8SIan Munsie 356f204e0b8SIan Munsie 357f204e0b8SIan Munsie static void assign_psn_space(struct cxl_context *ctx) 358f204e0b8SIan Munsie { 359f204e0b8SIan Munsie if (!ctx->afu->pp_size || ctx->master) { 360f204e0b8SIan Munsie ctx->psn_phys = ctx->afu->psn_phys; 361f204e0b8SIan Munsie ctx->psn_size = ctx->afu->adapter->ps_size; 362f204e0b8SIan Munsie } else { 363f204e0b8SIan Munsie ctx->psn_phys = ctx->afu->psn_phys + 364f204e0b8SIan Munsie (ctx->afu->pp_offset + ctx->afu->pp_size * ctx->pe); 365f204e0b8SIan Munsie ctx->psn_size = ctx->afu->pp_size; 366f204e0b8SIan Munsie } 367f204e0b8SIan Munsie } 368f204e0b8SIan Munsie 369f204e0b8SIan Munsie static int activate_afu_directed(struct cxl_afu *afu) 370f204e0b8SIan Munsie { 371f204e0b8SIan Munsie int rc; 372f204e0b8SIan Munsie 373f204e0b8SIan Munsie dev_info(&afu->dev, "Activating AFU directed mode\n"); 374f204e0b8SIan Munsie 375f204e0b8SIan Munsie if (alloc_spa(afu)) 376f204e0b8SIan Munsie return -ENOMEM; 377f204e0b8SIan Munsie 378f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU); 379f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); 380f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L); 381f204e0b8SIan Munsie 382f204e0b8SIan Munsie afu->current_mode = CXL_MODE_DIRECTED; 383f204e0b8SIan Munsie afu->num_procs = afu->max_procs_virtualised; 384f204e0b8SIan Munsie 385f204e0b8SIan Munsie if ((rc = cxl_chardev_m_afu_add(afu))) 386f204e0b8SIan Munsie return rc; 387f204e0b8SIan Munsie 388f204e0b8SIan Munsie if ((rc = cxl_sysfs_afu_m_add(afu))) 389f204e0b8SIan Munsie goto err; 390f204e0b8SIan Munsie 391f204e0b8SIan Munsie if ((rc = cxl_chardev_s_afu_add(afu))) 392f204e0b8SIan Munsie goto err1; 393f204e0b8SIan Munsie 394f204e0b8SIan Munsie return 0; 395f204e0b8SIan Munsie err1: 396f204e0b8SIan Munsie cxl_sysfs_afu_m_remove(afu); 397f204e0b8SIan Munsie err: 398f204e0b8SIan Munsie cxl_chardev_afu_remove(afu); 399f204e0b8SIan Munsie return rc; 400f204e0b8SIan Munsie } 401f204e0b8SIan Munsie 402f204e0b8SIan Munsie #ifdef CONFIG_CPU_LITTLE_ENDIAN 403f204e0b8SIan Munsie #define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE) 404f204e0b8SIan Munsie #else 405f204e0b8SIan Munsie #define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE)) 406f204e0b8SIan Munsie #endif 407f204e0b8SIan Munsie 408f204e0b8SIan Munsie static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr) 409f204e0b8SIan Munsie { 410f204e0b8SIan Munsie u64 sr; 411f204e0b8SIan Munsie int r, result; 412f204e0b8SIan Munsie 413f204e0b8SIan Munsie assign_psn_space(ctx); 414f204e0b8SIan Munsie 415f204e0b8SIan Munsie ctx->elem->ctxtime = 0; /* disable */ 416f204e0b8SIan Munsie ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID)); 417f204e0b8SIan Munsie ctx->elem->haurp = 0; /* disable */ 418f204e0b8SIan Munsie ctx->elem->sdr = cpu_to_be64(mfspr(SPRN_SDR1)); 419f204e0b8SIan Munsie 4205100a9d6SIan Munsie sr = 0; 421f204e0b8SIan Munsie if (ctx->master) 422f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_MP; 423f204e0b8SIan Munsie if (mfspr(SPRN_LPCR) & LPCR_TC) 424f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_TC; 425f204e0b8SIan Munsie /* HV=0, PR=1, R=1 for userspace 426f204e0b8SIan Munsie * For kernel contexts: this would need to change 427f204e0b8SIan Munsie */ 428f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R; 429f204e0b8SIan Munsie set_endian(sr); 430f204e0b8SIan Munsie sr &= ~(CXL_PSL_SR_An_HV); 431f204e0b8SIan Munsie if (!test_tsk_thread_flag(current, TIF_32BIT)) 432f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_SF; 433f204e0b8SIan Munsie ctx->elem->common.pid = cpu_to_be32(current->pid); 434f204e0b8SIan Munsie ctx->elem->common.tid = 0; 435f204e0b8SIan Munsie ctx->elem->sr = cpu_to_be64(sr); 436f204e0b8SIan Munsie 437f204e0b8SIan Munsie ctx->elem->common.csrp = 0; /* disable */ 438f204e0b8SIan Munsie ctx->elem->common.aurp0 = 0; /* disable */ 439f204e0b8SIan Munsie ctx->elem->common.aurp1 = 0; /* disable */ 440f204e0b8SIan Munsie 441f204e0b8SIan Munsie cxl_prefault(ctx, wed); 442f204e0b8SIan Munsie 443f204e0b8SIan Munsie ctx->elem->common.sstp0 = cpu_to_be64(ctx->sstp0); 444f204e0b8SIan Munsie ctx->elem->common.sstp1 = cpu_to_be64(ctx->sstp1); 445f204e0b8SIan Munsie 446f204e0b8SIan Munsie for (r = 0; r < CXL_IRQ_RANGES; r++) { 447f204e0b8SIan Munsie ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]); 448f204e0b8SIan Munsie ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]); 449f204e0b8SIan Munsie } 450f204e0b8SIan Munsie 451f204e0b8SIan Munsie ctx->elem->common.amr = cpu_to_be64(amr); 452f204e0b8SIan Munsie ctx->elem->common.wed = cpu_to_be64(wed); 453f204e0b8SIan Munsie 454f204e0b8SIan Munsie /* first guy needs to enable */ 455f204e0b8SIan Munsie if ((result = afu_check_and_enable(ctx->afu))) 456f204e0b8SIan Munsie return result; 457f204e0b8SIan Munsie 458f204e0b8SIan Munsie add_process_element(ctx); 459f204e0b8SIan Munsie 460f204e0b8SIan Munsie return 0; 461f204e0b8SIan Munsie } 462f204e0b8SIan Munsie 463f204e0b8SIan Munsie static int deactivate_afu_directed(struct cxl_afu *afu) 464f204e0b8SIan Munsie { 465f204e0b8SIan Munsie dev_info(&afu->dev, "Deactivating AFU directed mode\n"); 466f204e0b8SIan Munsie 467f204e0b8SIan Munsie afu->current_mode = 0; 468f204e0b8SIan Munsie afu->num_procs = 0; 469f204e0b8SIan Munsie 470f204e0b8SIan Munsie cxl_sysfs_afu_m_remove(afu); 471f204e0b8SIan Munsie cxl_chardev_afu_remove(afu); 472f204e0b8SIan Munsie 473f204e0b8SIan Munsie cxl_afu_reset(afu); 474f204e0b8SIan Munsie cxl_afu_disable(afu); 475f204e0b8SIan Munsie cxl_psl_purge(afu); 476f204e0b8SIan Munsie 477f204e0b8SIan Munsie release_spa(afu); 478f204e0b8SIan Munsie 479f204e0b8SIan Munsie return 0; 480f204e0b8SIan Munsie } 481f204e0b8SIan Munsie 482f204e0b8SIan Munsie static int activate_dedicated_process(struct cxl_afu *afu) 483f204e0b8SIan Munsie { 484f204e0b8SIan Munsie dev_info(&afu->dev, "Activating dedicated process mode\n"); 485f204e0b8SIan Munsie 486f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process); 487f204e0b8SIan Munsie 488f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */ 489f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); /* disable */ 490f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); 491f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID)); 492f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_HAURP_An, 0); /* disable */ 493f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1)); 494f204e0b8SIan Munsie 495f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_CSRP_An, 0); /* disable */ 496f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_AURP0_An, 0); /* disable */ 497f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_AURP1_An, 0); /* disable */ 498f204e0b8SIan Munsie 499f204e0b8SIan Munsie afu->current_mode = CXL_MODE_DEDICATED; 500f204e0b8SIan Munsie afu->num_procs = 1; 501f204e0b8SIan Munsie 502f204e0b8SIan Munsie return cxl_chardev_d_afu_add(afu); 503f204e0b8SIan Munsie } 504f204e0b8SIan Munsie 505f204e0b8SIan Munsie static int attach_dedicated(struct cxl_context *ctx, u64 wed, u64 amr) 506f204e0b8SIan Munsie { 507f204e0b8SIan Munsie struct cxl_afu *afu = ctx->afu; 508f204e0b8SIan Munsie u64 sr; 509f204e0b8SIan Munsie int rc; 510f204e0b8SIan Munsie 5115100a9d6SIan Munsie sr = 0; 512f204e0b8SIan Munsie set_endian(sr); 513f204e0b8SIan Munsie if (ctx->master) 514f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_MP; 515f204e0b8SIan Munsie if (mfspr(SPRN_LPCR) & LPCR_TC) 516f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_TC; 517f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R; 518f204e0b8SIan Munsie if (!test_tsk_thread_flag(current, TIF_32BIT)) 519f204e0b8SIan Munsie sr |= CXL_PSL_SR_An_SF; 520f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_PID_TID_An, (u64)current->pid << 32); 521f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_SR_An, sr); 522f204e0b8SIan Munsie 523f204e0b8SIan Munsie if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1))) 524f204e0b8SIan Munsie return rc; 525f204e0b8SIan Munsie 526f204e0b8SIan Munsie cxl_prefault(ctx, wed); 527f204e0b8SIan Munsie 528f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, 529f204e0b8SIan Munsie (((u64)ctx->irqs.offset[0] & 0xffff) << 48) | 530f204e0b8SIan Munsie (((u64)ctx->irqs.offset[1] & 0xffff) << 32) | 531f204e0b8SIan Munsie (((u64)ctx->irqs.offset[2] & 0xffff) << 16) | 532f204e0b8SIan Munsie ((u64)ctx->irqs.offset[3] & 0xffff)); 533f204e0b8SIan Munsie cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64) 534f204e0b8SIan Munsie (((u64)ctx->irqs.range[0] & 0xffff) << 48) | 535f204e0b8SIan Munsie (((u64)ctx->irqs.range[1] & 0xffff) << 32) | 536f204e0b8SIan Munsie (((u64)ctx->irqs.range[2] & 0xffff) << 16) | 537f204e0b8SIan Munsie ((u64)ctx->irqs.range[3] & 0xffff)); 538f204e0b8SIan Munsie 539f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_AMR_An, amr); 540f204e0b8SIan Munsie 541f204e0b8SIan Munsie /* master only context for dedicated */ 542f204e0b8SIan Munsie assign_psn_space(ctx); 543f204e0b8SIan Munsie 544f204e0b8SIan Munsie if ((rc = cxl_afu_reset(afu))) 545f204e0b8SIan Munsie return rc; 546f204e0b8SIan Munsie 547f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_WED_An, wed); 548f204e0b8SIan Munsie 549f204e0b8SIan Munsie return afu_enable(afu); 550f204e0b8SIan Munsie } 551f204e0b8SIan Munsie 552f204e0b8SIan Munsie static int deactivate_dedicated_process(struct cxl_afu *afu) 553f204e0b8SIan Munsie { 554f204e0b8SIan Munsie dev_info(&afu->dev, "Deactivating dedicated process mode\n"); 555f204e0b8SIan Munsie 556f204e0b8SIan Munsie afu->current_mode = 0; 557f204e0b8SIan Munsie afu->num_procs = 0; 558f204e0b8SIan Munsie 559f204e0b8SIan Munsie cxl_chardev_afu_remove(afu); 560f204e0b8SIan Munsie 561f204e0b8SIan Munsie return 0; 562f204e0b8SIan Munsie } 563f204e0b8SIan Munsie 564f204e0b8SIan Munsie int _cxl_afu_deactivate_mode(struct cxl_afu *afu, int mode) 565f204e0b8SIan Munsie { 566f204e0b8SIan Munsie if (mode == CXL_MODE_DIRECTED) 567f204e0b8SIan Munsie return deactivate_afu_directed(afu); 568f204e0b8SIan Munsie if (mode == CXL_MODE_DEDICATED) 569f204e0b8SIan Munsie return deactivate_dedicated_process(afu); 570f204e0b8SIan Munsie return 0; 571f204e0b8SIan Munsie } 572f204e0b8SIan Munsie 573f204e0b8SIan Munsie int cxl_afu_deactivate_mode(struct cxl_afu *afu) 574f204e0b8SIan Munsie { 575f204e0b8SIan Munsie return _cxl_afu_deactivate_mode(afu, afu->current_mode); 576f204e0b8SIan Munsie } 577f204e0b8SIan Munsie 578f204e0b8SIan Munsie int cxl_afu_activate_mode(struct cxl_afu *afu, int mode) 579f204e0b8SIan Munsie { 580f204e0b8SIan Munsie if (!mode) 581f204e0b8SIan Munsie return 0; 582f204e0b8SIan Munsie if (!(mode & afu->modes_supported)) 583f204e0b8SIan Munsie return -EINVAL; 584f204e0b8SIan Munsie 585f204e0b8SIan Munsie if (mode == CXL_MODE_DIRECTED) 586f204e0b8SIan Munsie return activate_afu_directed(afu); 587f204e0b8SIan Munsie if (mode == CXL_MODE_DEDICATED) 588f204e0b8SIan Munsie return activate_dedicated_process(afu); 589f204e0b8SIan Munsie 590f204e0b8SIan Munsie return -EINVAL; 591f204e0b8SIan Munsie } 592f204e0b8SIan Munsie 593f204e0b8SIan Munsie int cxl_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr) 594f204e0b8SIan Munsie { 595f204e0b8SIan Munsie ctx->kernel = kernel; 596f204e0b8SIan Munsie if (ctx->afu->current_mode == CXL_MODE_DIRECTED) 597f204e0b8SIan Munsie return attach_afu_directed(ctx, wed, amr); 598f204e0b8SIan Munsie 599f204e0b8SIan Munsie if (ctx->afu->current_mode == CXL_MODE_DEDICATED) 600f204e0b8SIan Munsie return attach_dedicated(ctx, wed, amr); 601f204e0b8SIan Munsie 602f204e0b8SIan Munsie return -EINVAL; 603f204e0b8SIan Munsie } 604f204e0b8SIan Munsie 605f204e0b8SIan Munsie static inline int detach_process_native_dedicated(struct cxl_context *ctx) 606f204e0b8SIan Munsie { 607f204e0b8SIan Munsie cxl_afu_reset(ctx->afu); 608f204e0b8SIan Munsie cxl_afu_disable(ctx->afu); 609f204e0b8SIan Munsie cxl_psl_purge(ctx->afu); 610f204e0b8SIan Munsie return 0; 611f204e0b8SIan Munsie } 612f204e0b8SIan Munsie 613f204e0b8SIan Munsie /* 614f204e0b8SIan Munsie * TODO: handle case when this is called inside a rcu_read_lock() which may 615f204e0b8SIan Munsie * happen when we unbind the driver (ie. cxl_context_detach_all()) . Terminate 616f204e0b8SIan Munsie * & remove use a mutex lock and schedule which will not good with lock held. 617f204e0b8SIan Munsie * May need to write do_process_element_cmd() that handles outstanding page 618f204e0b8SIan Munsie * faults synchronously. 619f204e0b8SIan Munsie */ 620f204e0b8SIan Munsie static inline int detach_process_native_afu_directed(struct cxl_context *ctx) 621f204e0b8SIan Munsie { 622f204e0b8SIan Munsie if (!ctx->pe_inserted) 623f204e0b8SIan Munsie return 0; 624f204e0b8SIan Munsie if (terminate_process_element(ctx)) 625f204e0b8SIan Munsie return -1; 626f204e0b8SIan Munsie if (remove_process_element(ctx)) 627f204e0b8SIan Munsie return -1; 628f204e0b8SIan Munsie 629f204e0b8SIan Munsie return 0; 630f204e0b8SIan Munsie } 631f204e0b8SIan Munsie 632f204e0b8SIan Munsie int cxl_detach_process(struct cxl_context *ctx) 633f204e0b8SIan Munsie { 634f204e0b8SIan Munsie if (ctx->afu->current_mode == CXL_MODE_DEDICATED) 635f204e0b8SIan Munsie return detach_process_native_dedicated(ctx); 636f204e0b8SIan Munsie 637f204e0b8SIan Munsie return detach_process_native_afu_directed(ctx); 638f204e0b8SIan Munsie } 639f204e0b8SIan Munsie 640bc78b05bSIan Munsie int cxl_get_irq(struct cxl_afu *afu, struct cxl_irq_info *info) 641f204e0b8SIan Munsie { 642f204e0b8SIan Munsie u64 pidtid; 643f204e0b8SIan Munsie 644bc78b05bSIan Munsie info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 645bc78b05bSIan Munsie info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); 646bc78b05bSIan Munsie info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An); 647bc78b05bSIan Munsie pidtid = cxl_p2n_read(afu, CXL_PSL_PID_TID_An); 648f204e0b8SIan Munsie info->pid = pidtid >> 32; 649f204e0b8SIan Munsie info->tid = pidtid & 0xffffffff; 650bc78b05bSIan Munsie info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An); 651bc78b05bSIan Munsie info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An); 652f204e0b8SIan Munsie 653f204e0b8SIan Munsie return 0; 654f204e0b8SIan Munsie } 655f204e0b8SIan Munsie 656f204e0b8SIan Munsie static void recover_psl_err(struct cxl_afu *afu, u64 errstat) 657f204e0b8SIan Munsie { 658f204e0b8SIan Munsie u64 dsisr; 659f204e0b8SIan Munsie 660f204e0b8SIan Munsie pr_devel("RECOVERING FROM PSL ERROR... (0x%.16llx)\n", errstat); 661f204e0b8SIan Munsie 662f204e0b8SIan Munsie /* Clear PSL_DSISR[PE] */ 663f204e0b8SIan Munsie dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); 664f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE); 665f204e0b8SIan Munsie 666f204e0b8SIan Munsie /* Write 1s to clear error status bits */ 667f204e0b8SIan Munsie cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat); 668f204e0b8SIan Munsie } 669f204e0b8SIan Munsie 670f204e0b8SIan Munsie int cxl_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask) 671f204e0b8SIan Munsie { 672f204e0b8SIan Munsie if (tfc) 673f204e0b8SIan Munsie cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc); 674f204e0b8SIan Munsie if (psl_reset_mask) 675f204e0b8SIan Munsie recover_psl_err(ctx->afu, psl_reset_mask); 676f204e0b8SIan Munsie 677f204e0b8SIan Munsie return 0; 678f204e0b8SIan Munsie } 679f204e0b8SIan Munsie 680f204e0b8SIan Munsie int cxl_check_error(struct cxl_afu *afu) 681f204e0b8SIan Munsie { 682f204e0b8SIan Munsie return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL); 683f204e0b8SIan Munsie } 684