16914c757SFrederic Barrat // SPDX-License-Identifier: GPL-2.0+ 26914c757SFrederic Barrat // Copyright 2017 IBM Corp. 36914c757SFrederic Barrat #include <asm/pnv-ocxl.h> 46914c757SFrederic Barrat #include <asm/opal.h> 52cb3d64bSFrederic Barrat #include <misc/ocxl-config.h> 66914c757SFrederic Barrat #include "pci.h" 76914c757SFrederic Barrat 86914c757SFrederic Barrat #define PNV_OCXL_TL_P9_RECV_CAP 0x000000000000000Full 92cb3d64bSFrederic Barrat #define PNV_OCXL_ACTAG_MAX 64 106914c757SFrederic Barrat /* PASIDs are 20-bit, but on P9, NPU can only handle 15 bits */ 116914c757SFrederic Barrat #define PNV_OCXL_PASID_BITS 15 126914c757SFrederic Barrat #define PNV_OCXL_PASID_MAX ((1 << PNV_OCXL_PASID_BITS) - 1) 136914c757SFrederic Barrat 142cb3d64bSFrederic Barrat #define AFU_PRESENT (1 << 31) 152cb3d64bSFrederic Barrat #define AFU_INDEX_MASK 0x3F000000 162cb3d64bSFrederic Barrat #define AFU_INDEX_SHIFT 24 172cb3d64bSFrederic Barrat #define ACTAG_MASK 0xFFF 182cb3d64bSFrederic Barrat 192cb3d64bSFrederic Barrat 202cb3d64bSFrederic Barrat struct actag_range { 212cb3d64bSFrederic Barrat u16 start; 222cb3d64bSFrederic Barrat u16 count; 232cb3d64bSFrederic Barrat }; 242cb3d64bSFrederic Barrat 252cb3d64bSFrederic Barrat struct npu_link { 262cb3d64bSFrederic Barrat struct list_head list; 272cb3d64bSFrederic Barrat int domain; 282cb3d64bSFrederic Barrat int bus; 292cb3d64bSFrederic Barrat int dev; 302cb3d64bSFrederic Barrat u16 fn_desired_actags[8]; 312cb3d64bSFrederic Barrat struct actag_range fn_actags[8]; 322cb3d64bSFrederic Barrat bool assignment_done; 332cb3d64bSFrederic Barrat }; 342cb3d64bSFrederic Barrat static struct list_head links_list = LIST_HEAD_INIT(links_list); 352cb3d64bSFrederic Barrat static DEFINE_MUTEX(links_list_lock); 362cb3d64bSFrederic Barrat 372cb3d64bSFrederic Barrat 382cb3d64bSFrederic Barrat /* 392cb3d64bSFrederic Barrat * opencapi actags handling: 402cb3d64bSFrederic Barrat * 412cb3d64bSFrederic Barrat * When sending commands, the opencapi device references the memory 422cb3d64bSFrederic Barrat * context it's targeting with an 'actag', which is really an alias 432cb3d64bSFrederic Barrat * for a (BDF, pasid) combination. When it receives a command, the NPU 442cb3d64bSFrederic Barrat * must do a lookup of the actag to identify the memory context. The 452cb3d64bSFrederic Barrat * hardware supports a finite number of actags per link (64 for 462cb3d64bSFrederic Barrat * POWER9). 472cb3d64bSFrederic Barrat * 482cb3d64bSFrederic Barrat * The device can carry multiple functions, and each function can have 492cb3d64bSFrederic Barrat * multiple AFUs. Each AFU advertises in its config space the number 502cb3d64bSFrederic Barrat * of desired actags. The host must configure in the config space of 512cb3d64bSFrederic Barrat * the AFU how many actags the AFU is really allowed to use (which can 522cb3d64bSFrederic Barrat * be less than what the AFU desires). 532cb3d64bSFrederic Barrat * 542cb3d64bSFrederic Barrat * When a PCI function is probed by the driver, it has no visibility 552cb3d64bSFrederic Barrat * about the other PCI functions and how many actags they'd like, 562cb3d64bSFrederic Barrat * which makes it impossible to distribute actags fairly among AFUs. 572cb3d64bSFrederic Barrat * 582cb3d64bSFrederic Barrat * Unfortunately, the only way to know how many actags a function 592cb3d64bSFrederic Barrat * desires is by looking at the data for each AFU in the config space 602cb3d64bSFrederic Barrat * and add them up. Similarly, the only way to know how many actags 612cb3d64bSFrederic Barrat * all the functions of the physical device desire is by adding the 622cb3d64bSFrederic Barrat * previously computed function counts. Then we can match that against 632cb3d64bSFrederic Barrat * what the hardware supports. 642cb3d64bSFrederic Barrat * 652cb3d64bSFrederic Barrat * To get a comprehensive view, we use a 'pci fixup': at the end of 662cb3d64bSFrederic Barrat * PCI enumeration, each function counts how many actags its AFUs 672cb3d64bSFrederic Barrat * desire and we save it in a 'npu_link' structure, shared between all 682cb3d64bSFrederic Barrat * the PCI functions of a same device. Therefore, when the first 692cb3d64bSFrederic Barrat * function is probed by the driver, we can get an idea of the total 702cb3d64bSFrederic Barrat * count of desired actags for the device, and assign the actags to 712cb3d64bSFrederic Barrat * the AFUs, by pro-rating if needed. 722cb3d64bSFrederic Barrat */ 732cb3d64bSFrederic Barrat 742cb3d64bSFrederic Barrat static int find_dvsec_from_pos(struct pci_dev *dev, int dvsec_id, int pos) 752cb3d64bSFrederic Barrat { 762cb3d64bSFrederic Barrat int vsec = pos; 772cb3d64bSFrederic Barrat u16 vendor, id; 782cb3d64bSFrederic Barrat 792cb3d64bSFrederic Barrat while ((vsec = pci_find_next_ext_capability(dev, vsec, 802cb3d64bSFrederic Barrat OCXL_EXT_CAP_ID_DVSEC))) { 812cb3d64bSFrederic Barrat pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET, 822cb3d64bSFrederic Barrat &vendor); 832cb3d64bSFrederic Barrat pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id); 842cb3d64bSFrederic Barrat if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id) 852cb3d64bSFrederic Barrat return vsec; 862cb3d64bSFrederic Barrat } 872cb3d64bSFrederic Barrat return 0; 882cb3d64bSFrederic Barrat } 892cb3d64bSFrederic Barrat 902cb3d64bSFrederic Barrat static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx) 912cb3d64bSFrederic Barrat { 922cb3d64bSFrederic Barrat int vsec = 0; 932cb3d64bSFrederic Barrat u8 idx; 942cb3d64bSFrederic Barrat 952cb3d64bSFrederic Barrat while ((vsec = find_dvsec_from_pos(dev, OCXL_DVSEC_AFU_CTRL_ID, 962cb3d64bSFrederic Barrat vsec))) { 972cb3d64bSFrederic Barrat pci_read_config_byte(dev, vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX, 982cb3d64bSFrederic Barrat &idx); 992cb3d64bSFrederic Barrat if (idx == afu_idx) 1002cb3d64bSFrederic Barrat return vsec; 1012cb3d64bSFrederic Barrat } 1022cb3d64bSFrederic Barrat return 0; 1032cb3d64bSFrederic Barrat } 1042cb3d64bSFrederic Barrat 1052cb3d64bSFrederic Barrat static int get_max_afu_index(struct pci_dev *dev, int *afu_idx) 1062cb3d64bSFrederic Barrat { 1072cb3d64bSFrederic Barrat int pos; 1082cb3d64bSFrederic Barrat u32 val; 1092cb3d64bSFrederic Barrat 1102cb3d64bSFrederic Barrat pos = find_dvsec_from_pos(dev, OCXL_DVSEC_FUNC_ID, 0); 1112cb3d64bSFrederic Barrat if (!pos) 1122cb3d64bSFrederic Barrat return -ESRCH; 1132cb3d64bSFrederic Barrat 1142cb3d64bSFrederic Barrat pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val); 1152cb3d64bSFrederic Barrat if (val & AFU_PRESENT) 1162cb3d64bSFrederic Barrat *afu_idx = (val & AFU_INDEX_MASK) >> AFU_INDEX_SHIFT; 1172cb3d64bSFrederic Barrat else 1182cb3d64bSFrederic Barrat *afu_idx = -1; 1192cb3d64bSFrederic Barrat return 0; 1202cb3d64bSFrederic Barrat } 1212cb3d64bSFrederic Barrat 1222cb3d64bSFrederic Barrat static int get_actag_count(struct pci_dev *dev, int afu_idx, int *actag) 1232cb3d64bSFrederic Barrat { 1242cb3d64bSFrederic Barrat int pos; 1252cb3d64bSFrederic Barrat u16 actag_sup; 1262cb3d64bSFrederic Barrat 1272cb3d64bSFrederic Barrat pos = find_dvsec_afu_ctrl(dev, afu_idx); 1282cb3d64bSFrederic Barrat if (!pos) 1292cb3d64bSFrederic Barrat return -ESRCH; 1302cb3d64bSFrederic Barrat 1312cb3d64bSFrederic Barrat pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP, 1322cb3d64bSFrederic Barrat &actag_sup); 1332cb3d64bSFrederic Barrat *actag = actag_sup & ACTAG_MASK; 1342cb3d64bSFrederic Barrat return 0; 1352cb3d64bSFrederic Barrat } 1362cb3d64bSFrederic Barrat 1372cb3d64bSFrederic Barrat static struct npu_link *find_link(struct pci_dev *dev) 1382cb3d64bSFrederic Barrat { 1392cb3d64bSFrederic Barrat struct npu_link *link; 1402cb3d64bSFrederic Barrat 1412cb3d64bSFrederic Barrat list_for_each_entry(link, &links_list, list) { 1422cb3d64bSFrederic Barrat /* The functions of a device all share the same link */ 1432cb3d64bSFrederic Barrat if (link->domain == pci_domain_nr(dev->bus) && 1442cb3d64bSFrederic Barrat link->bus == dev->bus->number && 1452cb3d64bSFrederic Barrat link->dev == PCI_SLOT(dev->devfn)) { 1462cb3d64bSFrederic Barrat return link; 1472cb3d64bSFrederic Barrat } 1482cb3d64bSFrederic Barrat } 1492cb3d64bSFrederic Barrat 1502cb3d64bSFrederic Barrat /* link doesn't exist yet. Allocate one */ 1512cb3d64bSFrederic Barrat link = kzalloc(sizeof(struct npu_link), GFP_KERNEL); 1522cb3d64bSFrederic Barrat if (!link) 1532cb3d64bSFrederic Barrat return NULL; 1542cb3d64bSFrederic Barrat link->domain = pci_domain_nr(dev->bus); 1552cb3d64bSFrederic Barrat link->bus = dev->bus->number; 1562cb3d64bSFrederic Barrat link->dev = PCI_SLOT(dev->devfn); 1572cb3d64bSFrederic Barrat list_add(&link->list, &links_list); 1582cb3d64bSFrederic Barrat return link; 1592cb3d64bSFrederic Barrat } 1602cb3d64bSFrederic Barrat 1612cb3d64bSFrederic Barrat static void pnv_ocxl_fixup_actag(struct pci_dev *dev) 1622cb3d64bSFrederic Barrat { 1632cb3d64bSFrederic Barrat struct pci_controller *hose = pci_bus_to_host(dev->bus); 1642cb3d64bSFrederic Barrat struct pnv_phb *phb = hose->private_data; 1652cb3d64bSFrederic Barrat struct npu_link *link; 1662cb3d64bSFrederic Barrat int rc, afu_idx = -1, i, actag; 1672cb3d64bSFrederic Barrat 1682cb3d64bSFrederic Barrat if (!machine_is(powernv)) 1692cb3d64bSFrederic Barrat return; 1702cb3d64bSFrederic Barrat 1712cb3d64bSFrederic Barrat if (phb->type != PNV_PHB_NPU_OCAPI) 1722cb3d64bSFrederic Barrat return; 1732cb3d64bSFrederic Barrat 1742cb3d64bSFrederic Barrat mutex_lock(&links_list_lock); 1752cb3d64bSFrederic Barrat 1762cb3d64bSFrederic Barrat link = find_link(dev); 1772cb3d64bSFrederic Barrat if (!link) { 1782cb3d64bSFrederic Barrat dev_warn(&dev->dev, "couldn't update actag information\n"); 1792cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 1802cb3d64bSFrederic Barrat return; 1812cb3d64bSFrederic Barrat } 1822cb3d64bSFrederic Barrat 1832cb3d64bSFrederic Barrat /* 1842cb3d64bSFrederic Barrat * Check how many actags are desired for the AFUs under that 1852cb3d64bSFrederic Barrat * function and add it to the count for the link 1862cb3d64bSFrederic Barrat */ 1872cb3d64bSFrederic Barrat rc = get_max_afu_index(dev, &afu_idx); 1882cb3d64bSFrederic Barrat if (rc) { 1892cb3d64bSFrederic Barrat /* Most likely an invalid config space */ 1902cb3d64bSFrederic Barrat dev_dbg(&dev->dev, "couldn't find AFU information\n"); 1912cb3d64bSFrederic Barrat afu_idx = -1; 1922cb3d64bSFrederic Barrat } 1932cb3d64bSFrederic Barrat 1942cb3d64bSFrederic Barrat link->fn_desired_actags[PCI_FUNC(dev->devfn)] = 0; 1952cb3d64bSFrederic Barrat for (i = 0; i <= afu_idx; i++) { 1962cb3d64bSFrederic Barrat /* 1972cb3d64bSFrederic Barrat * AFU index 'holes' are allowed. So don't fail if we 1982cb3d64bSFrederic Barrat * can't read the actag info for an index 1992cb3d64bSFrederic Barrat */ 2002cb3d64bSFrederic Barrat rc = get_actag_count(dev, i, &actag); 2012cb3d64bSFrederic Barrat if (rc) 2022cb3d64bSFrederic Barrat continue; 2032cb3d64bSFrederic Barrat link->fn_desired_actags[PCI_FUNC(dev->devfn)] += actag; 2042cb3d64bSFrederic Barrat } 2052cb3d64bSFrederic Barrat dev_dbg(&dev->dev, "total actags for function: %d\n", 2062cb3d64bSFrederic Barrat link->fn_desired_actags[PCI_FUNC(dev->devfn)]); 2072cb3d64bSFrederic Barrat 2082cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 2092cb3d64bSFrederic Barrat } 2102cb3d64bSFrederic Barrat DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pnv_ocxl_fixup_actag); 2112cb3d64bSFrederic Barrat 2122cb3d64bSFrederic Barrat static u16 assign_fn_actags(u16 desired, u16 total) 2132cb3d64bSFrederic Barrat { 2142cb3d64bSFrederic Barrat u16 count; 2152cb3d64bSFrederic Barrat 2162cb3d64bSFrederic Barrat if (total <= PNV_OCXL_ACTAG_MAX) 2172cb3d64bSFrederic Barrat count = desired; 2182cb3d64bSFrederic Barrat else 2192cb3d64bSFrederic Barrat count = PNV_OCXL_ACTAG_MAX * desired / total; 2202cb3d64bSFrederic Barrat 2212cb3d64bSFrederic Barrat return count; 2222cb3d64bSFrederic Barrat } 2232cb3d64bSFrederic Barrat 2242cb3d64bSFrederic Barrat static void assign_actags(struct npu_link *link) 2252cb3d64bSFrederic Barrat { 2262cb3d64bSFrederic Barrat u16 actag_count, range_start = 0, total_desired = 0; 2272cb3d64bSFrederic Barrat int i; 2282cb3d64bSFrederic Barrat 2292cb3d64bSFrederic Barrat for (i = 0; i < 8; i++) 2302cb3d64bSFrederic Barrat total_desired += link->fn_desired_actags[i]; 2312cb3d64bSFrederic Barrat 2322cb3d64bSFrederic Barrat for (i = 0; i < 8; i++) { 2332cb3d64bSFrederic Barrat if (link->fn_desired_actags[i]) { 2342cb3d64bSFrederic Barrat actag_count = assign_fn_actags( 2352cb3d64bSFrederic Barrat link->fn_desired_actags[i], 2362cb3d64bSFrederic Barrat total_desired); 2372cb3d64bSFrederic Barrat link->fn_actags[i].start = range_start; 2382cb3d64bSFrederic Barrat link->fn_actags[i].count = actag_count; 2392cb3d64bSFrederic Barrat range_start += actag_count; 2402cb3d64bSFrederic Barrat WARN_ON(range_start >= PNV_OCXL_ACTAG_MAX); 2412cb3d64bSFrederic Barrat } 2422cb3d64bSFrederic Barrat pr_debug("link %x:%x:%x fct %d actags: start=%d count=%d (desired=%d)\n", 2432cb3d64bSFrederic Barrat link->domain, link->bus, link->dev, i, 2442cb3d64bSFrederic Barrat link->fn_actags[i].start, link->fn_actags[i].count, 2452cb3d64bSFrederic Barrat link->fn_desired_actags[i]); 2462cb3d64bSFrederic Barrat } 2472cb3d64bSFrederic Barrat link->assignment_done = true; 2482cb3d64bSFrederic Barrat } 2492cb3d64bSFrederic Barrat 2502cb3d64bSFrederic Barrat int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, 2512cb3d64bSFrederic Barrat u16 *supported) 2522cb3d64bSFrederic Barrat { 2532cb3d64bSFrederic Barrat struct npu_link *link; 2542cb3d64bSFrederic Barrat 2552cb3d64bSFrederic Barrat mutex_lock(&links_list_lock); 2562cb3d64bSFrederic Barrat 2572cb3d64bSFrederic Barrat link = find_link(dev); 2582cb3d64bSFrederic Barrat if (!link) { 2592cb3d64bSFrederic Barrat dev_err(&dev->dev, "actag information not found\n"); 2602cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 2612cb3d64bSFrederic Barrat return -ENODEV; 2622cb3d64bSFrederic Barrat } 2632cb3d64bSFrederic Barrat /* 2642cb3d64bSFrederic Barrat * On p9, we only have 64 actags per link, so they must be 2652cb3d64bSFrederic Barrat * shared by all the functions of the same adapter. We counted 2662cb3d64bSFrederic Barrat * the desired actag counts during PCI enumeration, so that we 2672cb3d64bSFrederic Barrat * can allocate a pro-rated number of actags to each function. 2682cb3d64bSFrederic Barrat */ 2692cb3d64bSFrederic Barrat if (!link->assignment_done) 2702cb3d64bSFrederic Barrat assign_actags(link); 2712cb3d64bSFrederic Barrat 2722cb3d64bSFrederic Barrat *base = link->fn_actags[PCI_FUNC(dev->devfn)].start; 2732cb3d64bSFrederic Barrat *enabled = link->fn_actags[PCI_FUNC(dev->devfn)].count; 2742cb3d64bSFrederic Barrat *supported = link->fn_desired_actags[PCI_FUNC(dev->devfn)]; 2752cb3d64bSFrederic Barrat 2762cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 2772cb3d64bSFrederic Barrat return 0; 2782cb3d64bSFrederic Barrat } 2792cb3d64bSFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_actag); 2802cb3d64bSFrederic Barrat 2812cb3d64bSFrederic Barrat int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count) 2822cb3d64bSFrederic Barrat { 2832cb3d64bSFrederic Barrat struct npu_link *link; 2842cb3d64bSFrederic Barrat int i, rc = -EINVAL; 2852cb3d64bSFrederic Barrat 2862cb3d64bSFrederic Barrat /* 2872cb3d64bSFrederic Barrat * The number of PASIDs (process address space ID) which can 2882cb3d64bSFrederic Barrat * be used by a function depends on how many functions exist 2892cb3d64bSFrederic Barrat * on the device. The NPU needs to be configured to know how 2902cb3d64bSFrederic Barrat * many bits are available to PASIDs and how many are to be 2912cb3d64bSFrederic Barrat * used by the function BDF indentifier. 2922cb3d64bSFrederic Barrat * 2932cb3d64bSFrederic Barrat * We only support one AFU-carrying function for now. 2942cb3d64bSFrederic Barrat */ 2952cb3d64bSFrederic Barrat mutex_lock(&links_list_lock); 2962cb3d64bSFrederic Barrat 2972cb3d64bSFrederic Barrat link = find_link(dev); 2982cb3d64bSFrederic Barrat if (!link) { 2992cb3d64bSFrederic Barrat dev_err(&dev->dev, "actag information not found\n"); 3002cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 3012cb3d64bSFrederic Barrat return -ENODEV; 3022cb3d64bSFrederic Barrat } 3032cb3d64bSFrederic Barrat 3042cb3d64bSFrederic Barrat for (i = 0; i < 8; i++) 3052cb3d64bSFrederic Barrat if (link->fn_desired_actags[i] && (i == PCI_FUNC(dev->devfn))) { 3062cb3d64bSFrederic Barrat *count = PNV_OCXL_PASID_MAX; 3072cb3d64bSFrederic Barrat rc = 0; 3082cb3d64bSFrederic Barrat break; 3092cb3d64bSFrederic Barrat } 3102cb3d64bSFrederic Barrat 3112cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 3122cb3d64bSFrederic Barrat dev_dbg(&dev->dev, "%d PASIDs available for function\n", 3132cb3d64bSFrederic Barrat rc ? 0 : *count); 3142cb3d64bSFrederic Barrat return rc; 3152cb3d64bSFrederic Barrat } 3162cb3d64bSFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_pasid_count); 3176914c757SFrederic Barrat 3186914c757SFrederic Barrat static void set_templ_rate(unsigned int templ, unsigned int rate, char *buf) 3196914c757SFrederic Barrat { 3206914c757SFrederic Barrat int shift, idx; 3216914c757SFrederic Barrat 3226914c757SFrederic Barrat WARN_ON(templ > PNV_OCXL_TL_MAX_TEMPLATE); 3236914c757SFrederic Barrat idx = (PNV_OCXL_TL_MAX_TEMPLATE - templ) / 2; 3246914c757SFrederic Barrat shift = 4 * (1 - ((PNV_OCXL_TL_MAX_TEMPLATE - templ) % 2)); 3256914c757SFrederic Barrat buf[idx] |= rate << shift; 3266914c757SFrederic Barrat } 3276914c757SFrederic Barrat 3286914c757SFrederic Barrat int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, 3296914c757SFrederic Barrat char *rate_buf, int rate_buf_size) 3306914c757SFrederic Barrat { 3316914c757SFrederic Barrat if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE) 3326914c757SFrederic Barrat return -EINVAL; 3336914c757SFrederic Barrat /* 3346914c757SFrederic Barrat * The TL capabilities are a characteristic of the NPU, so 3356914c757SFrederic Barrat * we go with hard-coded values. 3366914c757SFrederic Barrat * 3376914c757SFrederic Barrat * The receiving rate of each template is encoded on 4 bits. 3386914c757SFrederic Barrat * 3396914c757SFrederic Barrat * On P9: 3406914c757SFrederic Barrat * - templates 0 -> 3 are supported 3416914c757SFrederic Barrat * - templates 0, 1 and 3 have a 0 receiving rate 3426914c757SFrederic Barrat * - template 2 has receiving rate of 1 (extra cycle) 3436914c757SFrederic Barrat */ 3446914c757SFrederic Barrat memset(rate_buf, 0, rate_buf_size); 3456914c757SFrederic Barrat set_templ_rate(2, 1, rate_buf); 3466914c757SFrederic Barrat *cap = PNV_OCXL_TL_P9_RECV_CAP; 3476914c757SFrederic Barrat return 0; 3486914c757SFrederic Barrat } 3496914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_tl_cap); 3506914c757SFrederic Barrat 3516914c757SFrederic Barrat int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, 3526914c757SFrederic Barrat uint64_t rate_buf_phys, int rate_buf_size) 3536914c757SFrederic Barrat { 3546914c757SFrederic Barrat struct pci_controller *hose = pci_bus_to_host(dev->bus); 3556914c757SFrederic Barrat struct pnv_phb *phb = hose->private_data; 3566914c757SFrederic Barrat int rc; 3576914c757SFrederic Barrat 3586914c757SFrederic Barrat if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE) 3596914c757SFrederic Barrat return -EINVAL; 3606914c757SFrederic Barrat 3616914c757SFrederic Barrat rc = opal_npu_tl_set(phb->opal_id, dev->devfn, cap, 3626914c757SFrederic Barrat rate_buf_phys, rate_buf_size); 3636914c757SFrederic Barrat if (rc) { 3646914c757SFrederic Barrat dev_err(&dev->dev, "Can't configure host TL: %d\n", rc); 3656914c757SFrederic Barrat return -EINVAL; 3666914c757SFrederic Barrat } 3676914c757SFrederic Barrat return 0; 3686914c757SFrederic Barrat } 3696914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_set_tl_conf); 3706914c757SFrederic Barrat 3716914c757SFrederic Barrat int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq) 3726914c757SFrederic Barrat { 3736914c757SFrederic Barrat int rc; 3746914c757SFrederic Barrat 3756914c757SFrederic Barrat rc = of_property_read_u32(dev->dev.of_node, "ibm,opal-xsl-irq", hwirq); 3766914c757SFrederic Barrat if (rc) { 3776914c757SFrederic Barrat dev_err(&dev->dev, 3786914c757SFrederic Barrat "Can't get translation interrupt for device\n"); 3796914c757SFrederic Barrat return rc; 3806914c757SFrederic Barrat } 3816914c757SFrederic Barrat return 0; 3826914c757SFrederic Barrat } 3836914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_xsl_irq); 3846914c757SFrederic Barrat 3856914c757SFrederic Barrat void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, 3866914c757SFrederic Barrat void __iomem *tfc, void __iomem *pe_handle) 3876914c757SFrederic Barrat { 3886914c757SFrederic Barrat iounmap(dsisr); 3896914c757SFrederic Barrat iounmap(dar); 3906914c757SFrederic Barrat iounmap(tfc); 3916914c757SFrederic Barrat iounmap(pe_handle); 3926914c757SFrederic Barrat } 3936914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_xsl_regs); 3946914c757SFrederic Barrat 3956914c757SFrederic Barrat int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, 3966914c757SFrederic Barrat void __iomem **dar, void __iomem **tfc, 3976914c757SFrederic Barrat void __iomem **pe_handle) 3986914c757SFrederic Barrat { 3996914c757SFrederic Barrat u64 reg; 4006914c757SFrederic Barrat int i, j, rc = 0; 4016914c757SFrederic Barrat void __iomem *regs[4]; 4026914c757SFrederic Barrat 4036914c757SFrederic Barrat /* 4046914c757SFrederic Barrat * opal stores the mmio addresses of the DSISR, DAR, TFC and 4056914c757SFrederic Barrat * PE_HANDLE registers in a device tree property, in that 4066914c757SFrederic Barrat * order 4076914c757SFrederic Barrat */ 4086914c757SFrederic Barrat for (i = 0; i < 4; i++) { 4096914c757SFrederic Barrat rc = of_property_read_u64_index(dev->dev.of_node, 4106914c757SFrederic Barrat "ibm,opal-xsl-mmio", i, ®); 4116914c757SFrederic Barrat if (rc) 4126914c757SFrederic Barrat break; 4136914c757SFrederic Barrat regs[i] = ioremap(reg, 8); 4146914c757SFrederic Barrat if (!regs[i]) { 4156914c757SFrederic Barrat rc = -EINVAL; 4166914c757SFrederic Barrat break; 4176914c757SFrederic Barrat } 4186914c757SFrederic Barrat } 4196914c757SFrederic Barrat if (rc) { 4206914c757SFrederic Barrat dev_err(&dev->dev, "Can't map translation mmio registers\n"); 4216914c757SFrederic Barrat for (j = i - 1; j >= 0; j--) 4226914c757SFrederic Barrat iounmap(regs[j]); 4236914c757SFrederic Barrat } else { 4246914c757SFrederic Barrat *dsisr = regs[0]; 4256914c757SFrederic Barrat *dar = regs[1]; 4266914c757SFrederic Barrat *tfc = regs[2]; 4276914c757SFrederic Barrat *pe_handle = regs[3]; 4286914c757SFrederic Barrat } 4296914c757SFrederic Barrat return rc; 4306914c757SFrederic Barrat } 4316914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_map_xsl_regs); 4326914c757SFrederic Barrat 4336914c757SFrederic Barrat struct spa_data { 4346914c757SFrederic Barrat u64 phb_opal_id; 4356914c757SFrederic Barrat u32 bdfn; 4366914c757SFrederic Barrat }; 4376914c757SFrederic Barrat 4386914c757SFrederic Barrat int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, 4396914c757SFrederic Barrat void **platform_data) 4406914c757SFrederic Barrat { 4416914c757SFrederic Barrat struct pci_controller *hose = pci_bus_to_host(dev->bus); 4426914c757SFrederic Barrat struct pnv_phb *phb = hose->private_data; 4436914c757SFrederic Barrat struct spa_data *data; 4446914c757SFrederic Barrat u32 bdfn; 4456914c757SFrederic Barrat int rc; 4466914c757SFrederic Barrat 4476914c757SFrederic Barrat data = kzalloc(sizeof(*data), GFP_KERNEL); 4486914c757SFrederic Barrat if (!data) 4496914c757SFrederic Barrat return -ENOMEM; 4506914c757SFrederic Barrat 4516914c757SFrederic Barrat bdfn = (dev->bus->number << 8) | dev->devfn; 4526914c757SFrederic Barrat rc = opal_npu_spa_setup(phb->opal_id, bdfn, virt_to_phys(spa_mem), 4536914c757SFrederic Barrat PE_mask); 4546914c757SFrederic Barrat if (rc) { 4556914c757SFrederic Barrat dev_err(&dev->dev, "Can't setup Shared Process Area: %d\n", rc); 4566914c757SFrederic Barrat kfree(data); 4576914c757SFrederic Barrat return rc; 4586914c757SFrederic Barrat } 4596914c757SFrederic Barrat data->phb_opal_id = phb->opal_id; 4606914c757SFrederic Barrat data->bdfn = bdfn; 4616914c757SFrederic Barrat *platform_data = (void *) data; 4626914c757SFrederic Barrat return 0; 4636914c757SFrederic Barrat } 4646914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_spa_setup); 4656914c757SFrederic Barrat 4666914c757SFrederic Barrat void pnv_ocxl_spa_release(void *platform_data) 4676914c757SFrederic Barrat { 4686914c757SFrederic Barrat struct spa_data *data = (struct spa_data *) platform_data; 4696914c757SFrederic Barrat int rc; 4706914c757SFrederic Barrat 4716914c757SFrederic Barrat rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0); 4726914c757SFrederic Barrat WARN_ON(rc); 4736914c757SFrederic Barrat kfree(data); 4746914c757SFrederic Barrat } 4756914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release); 4766914c757SFrederic Barrat 47719df3958SAlastair D'Silva int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle) 4786914c757SFrederic Barrat { 4796914c757SFrederic Barrat struct spa_data *data = (struct spa_data *) platform_data; 4806914c757SFrederic Barrat int rc; 4816914c757SFrederic Barrat 4826914c757SFrederic Barrat rc = opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle); 4836914c757SFrederic Barrat return rc; 4846914c757SFrederic Barrat } 48519df3958SAlastair D'Silva EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe_from_cache); 486fc1347b5SChristophe Lombard 487fc1347b5SChristophe Lombard int pnv_ocxl_map_lpar(struct pci_dev *dev, uint64_t lparid, 488fc1347b5SChristophe Lombard uint64_t lpcr, void __iomem **arva) 489fc1347b5SChristophe Lombard { 490fc1347b5SChristophe Lombard struct pci_controller *hose = pci_bus_to_host(dev->bus); 491fc1347b5SChristophe Lombard struct pnv_phb *phb = hose->private_data; 492fc1347b5SChristophe Lombard u64 mmio_atsd; 493fc1347b5SChristophe Lombard int rc; 494fc1347b5SChristophe Lombard 495fc1347b5SChristophe Lombard /* ATSD physical address. 496fc1347b5SChristophe Lombard * ATSD LAUNCH register: write access initiates a shoot down to 497fc1347b5SChristophe Lombard * initiate the TLB Invalidate command. 498fc1347b5SChristophe Lombard */ 499fc1347b5SChristophe Lombard rc = of_property_read_u64_index(hose->dn, "ibm,mmio-atsd", 500fc1347b5SChristophe Lombard 0, &mmio_atsd); 501fc1347b5SChristophe Lombard if (rc) { 502fc1347b5SChristophe Lombard dev_info(&dev->dev, "No available ATSD found\n"); 503fc1347b5SChristophe Lombard return rc; 504fc1347b5SChristophe Lombard } 505fc1347b5SChristophe Lombard 506fc1347b5SChristophe Lombard /* Assign a register set to a Logical Partition and MMIO ATSD 507fc1347b5SChristophe Lombard * LPARID register to the required value. 508fc1347b5SChristophe Lombard */ 509fc1347b5SChristophe Lombard rc = opal_npu_map_lpar(phb->opal_id, pci_dev_id(dev), 510fc1347b5SChristophe Lombard lparid, lpcr); 511fc1347b5SChristophe Lombard if (rc) { 512fc1347b5SChristophe Lombard dev_err(&dev->dev, "Error mapping device to LPAR: %d\n", rc); 513fc1347b5SChristophe Lombard return rc; 514fc1347b5SChristophe Lombard } 515fc1347b5SChristophe Lombard 516fc1347b5SChristophe Lombard *arva = ioremap(mmio_atsd, 24); 517fc1347b5SChristophe Lombard if (!(*arva)) { 518fc1347b5SChristophe Lombard dev_warn(&dev->dev, "ioremap failed - mmio_atsd: %#llx\n", mmio_atsd); 519fc1347b5SChristophe Lombard rc = -ENOMEM; 520fc1347b5SChristophe Lombard } 521fc1347b5SChristophe Lombard 522fc1347b5SChristophe Lombard return rc; 523fc1347b5SChristophe Lombard } 524fc1347b5SChristophe Lombard EXPORT_SYMBOL_GPL(pnv_ocxl_map_lpar); 525fc1347b5SChristophe Lombard 526fc1347b5SChristophe Lombard void pnv_ocxl_unmap_lpar(void __iomem *arva) 527fc1347b5SChristophe Lombard { 528fc1347b5SChristophe Lombard iounmap(arva); 529fc1347b5SChristophe Lombard } 530fc1347b5SChristophe Lombard EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_lpar); 531*19b311caSChristophe Lombard 532*19b311caSChristophe Lombard void pnv_ocxl_tlb_invalidate(void __iomem *arva, 533*19b311caSChristophe Lombard unsigned long pid, 534*19b311caSChristophe Lombard unsigned long addr, 535*19b311caSChristophe Lombard unsigned long page_size) 536*19b311caSChristophe Lombard { 537*19b311caSChristophe Lombard unsigned long timeout = jiffies + (HZ * PNV_OCXL_ATSD_TIMEOUT); 538*19b311caSChristophe Lombard u64 val = 0ull; 539*19b311caSChristophe Lombard int pend; 540*19b311caSChristophe Lombard u8 size; 541*19b311caSChristophe Lombard 542*19b311caSChristophe Lombard if (!(arva)) 543*19b311caSChristophe Lombard return; 544*19b311caSChristophe Lombard 545*19b311caSChristophe Lombard if (addr) { 546*19b311caSChristophe Lombard /* load Abbreviated Virtual Address register with 547*19b311caSChristophe Lombard * the necessary value 548*19b311caSChristophe Lombard */ 549*19b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_AVA_AVA, addr >> (63-51)); 550*19b311caSChristophe Lombard out_be64(arva + PNV_OCXL_ATSD_AVA, val); 551*19b311caSChristophe Lombard } 552*19b311caSChristophe Lombard 553*19b311caSChristophe Lombard /* Write access initiates a shoot down to initiate the 554*19b311caSChristophe Lombard * TLB Invalidate command 555*19b311caSChristophe Lombard */ 556*19b311caSChristophe Lombard val = PNV_OCXL_ATSD_LNCH_R; 557*19b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_RIC, 0b10); 558*19b311caSChristophe Lombard if (addr) 559*19b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_IS, 0b00); 560*19b311caSChristophe Lombard else { 561*19b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_IS, 0b01); 562*19b311caSChristophe Lombard val |= PNV_OCXL_ATSD_LNCH_OCAPI_SINGLETON; 563*19b311caSChristophe Lombard } 564*19b311caSChristophe Lombard val |= PNV_OCXL_ATSD_LNCH_PRS; 565*19b311caSChristophe Lombard /* Actual Page Size to be invalidated 566*19b311caSChristophe Lombard * 000 4KB 567*19b311caSChristophe Lombard * 101 64KB 568*19b311caSChristophe Lombard * 001 2MB 569*19b311caSChristophe Lombard * 010 1GB 570*19b311caSChristophe Lombard */ 571*19b311caSChristophe Lombard size = 0b101; 572*19b311caSChristophe Lombard if (page_size == 0x1000) 573*19b311caSChristophe Lombard size = 0b000; 574*19b311caSChristophe Lombard if (page_size == 0x200000) 575*19b311caSChristophe Lombard size = 0b001; 576*19b311caSChristophe Lombard if (page_size == 0x40000000) 577*19b311caSChristophe Lombard size = 0b010; 578*19b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_AP, size); 579*19b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_PID, pid); 580*19b311caSChristophe Lombard out_be64(arva + PNV_OCXL_ATSD_LNCH, val); 581*19b311caSChristophe Lombard 582*19b311caSChristophe Lombard /* Poll the ATSD status register to determine when the 583*19b311caSChristophe Lombard * TLB Invalidate has been completed. 584*19b311caSChristophe Lombard */ 585*19b311caSChristophe Lombard val = in_be64(arva + PNV_OCXL_ATSD_STAT); 586*19b311caSChristophe Lombard pend = val >> 63; 587*19b311caSChristophe Lombard 588*19b311caSChristophe Lombard while (pend) { 589*19b311caSChristophe Lombard if (time_after_eq(jiffies, timeout)) { 590*19b311caSChristophe Lombard pr_err("%s - Timeout while reading XTS MMIO ATSD status register (val=%#llx, pidr=0x%lx)\n", 591*19b311caSChristophe Lombard __func__, val, pid); 592*19b311caSChristophe Lombard return; 593*19b311caSChristophe Lombard } 594*19b311caSChristophe Lombard cpu_relax(); 595*19b311caSChristophe Lombard val = in_be64(arva + PNV_OCXL_ATSD_STAT); 596*19b311caSChristophe Lombard pend = val >> 63; 597*19b311caSChristophe Lombard } 598*19b311caSChristophe Lombard } 599*19b311caSChristophe Lombard EXPORT_SYMBOL_GPL(pnv_ocxl_tlb_invalidate); 600