16914c757SFrederic Barrat // SPDX-License-Identifier: GPL-2.0+ 26914c757SFrederic Barrat // Copyright 2017 IBM Corp. 36914c757SFrederic Barrat #include <asm/pnv-ocxl.h> 46914c757SFrederic Barrat #include <asm/opal.h> 52cb3d64bSFrederic Barrat #include <misc/ocxl-config.h> 66914c757SFrederic Barrat #include "pci.h" 76914c757SFrederic Barrat 86914c757SFrederic Barrat #define PNV_OCXL_TL_P9_RECV_CAP 0x000000000000000Full 92cb3d64bSFrederic Barrat #define PNV_OCXL_ACTAG_MAX 64 106914c757SFrederic Barrat /* PASIDs are 20-bit, but on P9, NPU can only handle 15 bits */ 116914c757SFrederic Barrat #define PNV_OCXL_PASID_BITS 15 126914c757SFrederic Barrat #define PNV_OCXL_PASID_MAX ((1 << PNV_OCXL_PASID_BITS) - 1) 136914c757SFrederic Barrat 142cb3d64bSFrederic Barrat #define AFU_PRESENT (1 << 31) 152cb3d64bSFrederic Barrat #define AFU_INDEX_MASK 0x3F000000 162cb3d64bSFrederic Barrat #define AFU_INDEX_SHIFT 24 172cb3d64bSFrederic Barrat #define ACTAG_MASK 0xFFF 182cb3d64bSFrederic Barrat 192cb3d64bSFrederic Barrat 202cb3d64bSFrederic Barrat struct actag_range { 212cb3d64bSFrederic Barrat u16 start; 222cb3d64bSFrederic Barrat u16 count; 232cb3d64bSFrederic Barrat }; 242cb3d64bSFrederic Barrat 252cb3d64bSFrederic Barrat struct npu_link { 262cb3d64bSFrederic Barrat struct list_head list; 272cb3d64bSFrederic Barrat int domain; 282cb3d64bSFrederic Barrat int bus; 292cb3d64bSFrederic Barrat int dev; 302cb3d64bSFrederic Barrat u16 fn_desired_actags[8]; 312cb3d64bSFrederic Barrat struct actag_range fn_actags[8]; 322cb3d64bSFrederic Barrat bool assignment_done; 332cb3d64bSFrederic Barrat }; 342cb3d64bSFrederic Barrat static struct list_head links_list = LIST_HEAD_INIT(links_list); 352cb3d64bSFrederic Barrat static DEFINE_MUTEX(links_list_lock); 362cb3d64bSFrederic Barrat 372cb3d64bSFrederic Barrat 382cb3d64bSFrederic Barrat /* 392cb3d64bSFrederic Barrat * opencapi actags handling: 402cb3d64bSFrederic Barrat * 412cb3d64bSFrederic Barrat * When sending commands, the opencapi device references the memory 422cb3d64bSFrederic Barrat * context it's targeting with an 'actag', which is really an alias 432cb3d64bSFrederic Barrat * for a (BDF, pasid) combination. When it receives a command, the NPU 442cb3d64bSFrederic Barrat * must do a lookup of the actag to identify the memory context. The 452cb3d64bSFrederic Barrat * hardware supports a finite number of actags per link (64 for 462cb3d64bSFrederic Barrat * POWER9). 472cb3d64bSFrederic Barrat * 482cb3d64bSFrederic Barrat * The device can carry multiple functions, and each function can have 492cb3d64bSFrederic Barrat * multiple AFUs. Each AFU advertises in its config space the number 502cb3d64bSFrederic Barrat * of desired actags. The host must configure in the config space of 512cb3d64bSFrederic Barrat * the AFU how many actags the AFU is really allowed to use (which can 522cb3d64bSFrederic Barrat * be less than what the AFU desires). 532cb3d64bSFrederic Barrat * 542cb3d64bSFrederic Barrat * When a PCI function is probed by the driver, it has no visibility 552cb3d64bSFrederic Barrat * about the other PCI functions and how many actags they'd like, 562cb3d64bSFrederic Barrat * which makes it impossible to distribute actags fairly among AFUs. 572cb3d64bSFrederic Barrat * 582cb3d64bSFrederic Barrat * Unfortunately, the only way to know how many actags a function 592cb3d64bSFrederic Barrat * desires is by looking at the data for each AFU in the config space 602cb3d64bSFrederic Barrat * and add them up. Similarly, the only way to know how many actags 612cb3d64bSFrederic Barrat * all the functions of the physical device desire is by adding the 622cb3d64bSFrederic Barrat * previously computed function counts. Then we can match that against 632cb3d64bSFrederic Barrat * what the hardware supports. 642cb3d64bSFrederic Barrat * 652cb3d64bSFrederic Barrat * To get a comprehensive view, we use a 'pci fixup': at the end of 662cb3d64bSFrederic Barrat * PCI enumeration, each function counts how many actags its AFUs 672cb3d64bSFrederic Barrat * desire and we save it in a 'npu_link' structure, shared between all 682cb3d64bSFrederic Barrat * the PCI functions of a same device. Therefore, when the first 692cb3d64bSFrederic Barrat * function is probed by the driver, we can get an idea of the total 702cb3d64bSFrederic Barrat * count of desired actags for the device, and assign the actags to 712cb3d64bSFrederic Barrat * the AFUs, by pro-rating if needed. 722cb3d64bSFrederic Barrat */ 732cb3d64bSFrederic Barrat 742cb3d64bSFrederic Barrat static int find_dvsec_from_pos(struct pci_dev *dev, int dvsec_id, int pos) 752cb3d64bSFrederic Barrat { 762cb3d64bSFrederic Barrat int vsec = pos; 772cb3d64bSFrederic Barrat u16 vendor, id; 782cb3d64bSFrederic Barrat 792cb3d64bSFrederic Barrat while ((vsec = pci_find_next_ext_capability(dev, vsec, 802cb3d64bSFrederic Barrat OCXL_EXT_CAP_ID_DVSEC))) { 812cb3d64bSFrederic Barrat pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET, 822cb3d64bSFrederic Barrat &vendor); 832cb3d64bSFrederic Barrat pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id); 842cb3d64bSFrederic Barrat if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id) 852cb3d64bSFrederic Barrat return vsec; 862cb3d64bSFrederic Barrat } 872cb3d64bSFrederic Barrat return 0; 882cb3d64bSFrederic Barrat } 892cb3d64bSFrederic Barrat 902cb3d64bSFrederic Barrat static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx) 912cb3d64bSFrederic Barrat { 922cb3d64bSFrederic Barrat int vsec = 0; 932cb3d64bSFrederic Barrat u8 idx; 942cb3d64bSFrederic Barrat 952cb3d64bSFrederic Barrat while ((vsec = find_dvsec_from_pos(dev, OCXL_DVSEC_AFU_CTRL_ID, 962cb3d64bSFrederic Barrat vsec))) { 972cb3d64bSFrederic Barrat pci_read_config_byte(dev, vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX, 982cb3d64bSFrederic Barrat &idx); 992cb3d64bSFrederic Barrat if (idx == afu_idx) 1002cb3d64bSFrederic Barrat return vsec; 1012cb3d64bSFrederic Barrat } 1022cb3d64bSFrederic Barrat return 0; 1032cb3d64bSFrederic Barrat } 1042cb3d64bSFrederic Barrat 1052cb3d64bSFrederic Barrat static int get_max_afu_index(struct pci_dev *dev, int *afu_idx) 1062cb3d64bSFrederic Barrat { 1072cb3d64bSFrederic Barrat int pos; 1082cb3d64bSFrederic Barrat u32 val; 1092cb3d64bSFrederic Barrat 110c6d7e134SBen Widawsky pos = pci_find_dvsec_capability(dev, PCI_VENDOR_ID_IBM, 111c6d7e134SBen Widawsky OCXL_DVSEC_FUNC_ID); 1122cb3d64bSFrederic Barrat if (!pos) 1132cb3d64bSFrederic Barrat return -ESRCH; 1142cb3d64bSFrederic Barrat 1152cb3d64bSFrederic Barrat pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val); 1162cb3d64bSFrederic Barrat if (val & AFU_PRESENT) 1172cb3d64bSFrederic Barrat *afu_idx = (val & AFU_INDEX_MASK) >> AFU_INDEX_SHIFT; 1182cb3d64bSFrederic Barrat else 1192cb3d64bSFrederic Barrat *afu_idx = -1; 1202cb3d64bSFrederic Barrat return 0; 1212cb3d64bSFrederic Barrat } 1222cb3d64bSFrederic Barrat 1232cb3d64bSFrederic Barrat static int get_actag_count(struct pci_dev *dev, int afu_idx, int *actag) 1242cb3d64bSFrederic Barrat { 1252cb3d64bSFrederic Barrat int pos; 1262cb3d64bSFrederic Barrat u16 actag_sup; 1272cb3d64bSFrederic Barrat 1282cb3d64bSFrederic Barrat pos = find_dvsec_afu_ctrl(dev, afu_idx); 1292cb3d64bSFrederic Barrat if (!pos) 1302cb3d64bSFrederic Barrat return -ESRCH; 1312cb3d64bSFrederic Barrat 1322cb3d64bSFrederic Barrat pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP, 1332cb3d64bSFrederic Barrat &actag_sup); 1342cb3d64bSFrederic Barrat *actag = actag_sup & ACTAG_MASK; 1352cb3d64bSFrederic Barrat return 0; 1362cb3d64bSFrederic Barrat } 1372cb3d64bSFrederic Barrat 1382cb3d64bSFrederic Barrat static struct npu_link *find_link(struct pci_dev *dev) 1392cb3d64bSFrederic Barrat { 1402cb3d64bSFrederic Barrat struct npu_link *link; 1412cb3d64bSFrederic Barrat 1422cb3d64bSFrederic Barrat list_for_each_entry(link, &links_list, list) { 1432cb3d64bSFrederic Barrat /* The functions of a device all share the same link */ 1442cb3d64bSFrederic Barrat if (link->domain == pci_domain_nr(dev->bus) && 1452cb3d64bSFrederic Barrat link->bus == dev->bus->number && 1462cb3d64bSFrederic Barrat link->dev == PCI_SLOT(dev->devfn)) { 1472cb3d64bSFrederic Barrat return link; 1482cb3d64bSFrederic Barrat } 1492cb3d64bSFrederic Barrat } 1502cb3d64bSFrederic Barrat 1512cb3d64bSFrederic Barrat /* link doesn't exist yet. Allocate one */ 1522cb3d64bSFrederic Barrat link = kzalloc(sizeof(struct npu_link), GFP_KERNEL); 1532cb3d64bSFrederic Barrat if (!link) 1542cb3d64bSFrederic Barrat return NULL; 1552cb3d64bSFrederic Barrat link->domain = pci_domain_nr(dev->bus); 1562cb3d64bSFrederic Barrat link->bus = dev->bus->number; 1572cb3d64bSFrederic Barrat link->dev = PCI_SLOT(dev->devfn); 1582cb3d64bSFrederic Barrat list_add(&link->list, &links_list); 1592cb3d64bSFrederic Barrat return link; 1602cb3d64bSFrederic Barrat } 1612cb3d64bSFrederic Barrat 1622cb3d64bSFrederic Barrat static void pnv_ocxl_fixup_actag(struct pci_dev *dev) 1632cb3d64bSFrederic Barrat { 1642cb3d64bSFrederic Barrat struct pci_controller *hose = pci_bus_to_host(dev->bus); 1652cb3d64bSFrederic Barrat struct pnv_phb *phb = hose->private_data; 1662cb3d64bSFrederic Barrat struct npu_link *link; 1672cb3d64bSFrederic Barrat int rc, afu_idx = -1, i, actag; 1682cb3d64bSFrederic Barrat 1692cb3d64bSFrederic Barrat if (!machine_is(powernv)) 1702cb3d64bSFrederic Barrat return; 1712cb3d64bSFrederic Barrat 1722cb3d64bSFrederic Barrat if (phb->type != PNV_PHB_NPU_OCAPI) 1732cb3d64bSFrederic Barrat return; 1742cb3d64bSFrederic Barrat 1752cb3d64bSFrederic Barrat mutex_lock(&links_list_lock); 1762cb3d64bSFrederic Barrat 1772cb3d64bSFrederic Barrat link = find_link(dev); 1782cb3d64bSFrederic Barrat if (!link) { 1792cb3d64bSFrederic Barrat dev_warn(&dev->dev, "couldn't update actag information\n"); 1802cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 1812cb3d64bSFrederic Barrat return; 1822cb3d64bSFrederic Barrat } 1832cb3d64bSFrederic Barrat 1842cb3d64bSFrederic Barrat /* 1852cb3d64bSFrederic Barrat * Check how many actags are desired for the AFUs under that 1862cb3d64bSFrederic Barrat * function and add it to the count for the link 1872cb3d64bSFrederic Barrat */ 1882cb3d64bSFrederic Barrat rc = get_max_afu_index(dev, &afu_idx); 1892cb3d64bSFrederic Barrat if (rc) { 1902cb3d64bSFrederic Barrat /* Most likely an invalid config space */ 1912cb3d64bSFrederic Barrat dev_dbg(&dev->dev, "couldn't find AFU information\n"); 1922cb3d64bSFrederic Barrat afu_idx = -1; 1932cb3d64bSFrederic Barrat } 1942cb3d64bSFrederic Barrat 1952cb3d64bSFrederic Barrat link->fn_desired_actags[PCI_FUNC(dev->devfn)] = 0; 1962cb3d64bSFrederic Barrat for (i = 0; i <= afu_idx; i++) { 1972cb3d64bSFrederic Barrat /* 1982cb3d64bSFrederic Barrat * AFU index 'holes' are allowed. So don't fail if we 1992cb3d64bSFrederic Barrat * can't read the actag info for an index 2002cb3d64bSFrederic Barrat */ 2012cb3d64bSFrederic Barrat rc = get_actag_count(dev, i, &actag); 2022cb3d64bSFrederic Barrat if (rc) 2032cb3d64bSFrederic Barrat continue; 2042cb3d64bSFrederic Barrat link->fn_desired_actags[PCI_FUNC(dev->devfn)] += actag; 2052cb3d64bSFrederic Barrat } 2062cb3d64bSFrederic Barrat dev_dbg(&dev->dev, "total actags for function: %d\n", 2072cb3d64bSFrederic Barrat link->fn_desired_actags[PCI_FUNC(dev->devfn)]); 2082cb3d64bSFrederic Barrat 2092cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 2102cb3d64bSFrederic Barrat } 2112cb3d64bSFrederic Barrat DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pnv_ocxl_fixup_actag); 2122cb3d64bSFrederic Barrat 2132cb3d64bSFrederic Barrat static u16 assign_fn_actags(u16 desired, u16 total) 2142cb3d64bSFrederic Barrat { 2152cb3d64bSFrederic Barrat u16 count; 2162cb3d64bSFrederic Barrat 2172cb3d64bSFrederic Barrat if (total <= PNV_OCXL_ACTAG_MAX) 2182cb3d64bSFrederic Barrat count = desired; 2192cb3d64bSFrederic Barrat else 2202cb3d64bSFrederic Barrat count = PNV_OCXL_ACTAG_MAX * desired / total; 2212cb3d64bSFrederic Barrat 2222cb3d64bSFrederic Barrat return count; 2232cb3d64bSFrederic Barrat } 2242cb3d64bSFrederic Barrat 2252cb3d64bSFrederic Barrat static void assign_actags(struct npu_link *link) 2262cb3d64bSFrederic Barrat { 2272cb3d64bSFrederic Barrat u16 actag_count, range_start = 0, total_desired = 0; 2282cb3d64bSFrederic Barrat int i; 2292cb3d64bSFrederic Barrat 2302cb3d64bSFrederic Barrat for (i = 0; i < 8; i++) 2312cb3d64bSFrederic Barrat total_desired += link->fn_desired_actags[i]; 2322cb3d64bSFrederic Barrat 2332cb3d64bSFrederic Barrat for (i = 0; i < 8; i++) { 2342cb3d64bSFrederic Barrat if (link->fn_desired_actags[i]) { 2352cb3d64bSFrederic Barrat actag_count = assign_fn_actags( 2362cb3d64bSFrederic Barrat link->fn_desired_actags[i], 2372cb3d64bSFrederic Barrat total_desired); 2382cb3d64bSFrederic Barrat link->fn_actags[i].start = range_start; 2392cb3d64bSFrederic Barrat link->fn_actags[i].count = actag_count; 2402cb3d64bSFrederic Barrat range_start += actag_count; 2412cb3d64bSFrederic Barrat WARN_ON(range_start >= PNV_OCXL_ACTAG_MAX); 2422cb3d64bSFrederic Barrat } 2432cb3d64bSFrederic Barrat pr_debug("link %x:%x:%x fct %d actags: start=%d count=%d (desired=%d)\n", 2442cb3d64bSFrederic Barrat link->domain, link->bus, link->dev, i, 2452cb3d64bSFrederic Barrat link->fn_actags[i].start, link->fn_actags[i].count, 2462cb3d64bSFrederic Barrat link->fn_desired_actags[i]); 2472cb3d64bSFrederic Barrat } 2482cb3d64bSFrederic Barrat link->assignment_done = true; 2492cb3d64bSFrederic Barrat } 2502cb3d64bSFrederic Barrat 2512cb3d64bSFrederic Barrat int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, 2522cb3d64bSFrederic Barrat u16 *supported) 2532cb3d64bSFrederic Barrat { 2542cb3d64bSFrederic Barrat struct npu_link *link; 2552cb3d64bSFrederic Barrat 2562cb3d64bSFrederic Barrat mutex_lock(&links_list_lock); 2572cb3d64bSFrederic Barrat 2582cb3d64bSFrederic Barrat link = find_link(dev); 2592cb3d64bSFrederic Barrat if (!link) { 2602cb3d64bSFrederic Barrat dev_err(&dev->dev, "actag information not found\n"); 2612cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 2622cb3d64bSFrederic Barrat return -ENODEV; 2632cb3d64bSFrederic Barrat } 2642cb3d64bSFrederic Barrat /* 2652cb3d64bSFrederic Barrat * On p9, we only have 64 actags per link, so they must be 2662cb3d64bSFrederic Barrat * shared by all the functions of the same adapter. We counted 2672cb3d64bSFrederic Barrat * the desired actag counts during PCI enumeration, so that we 2682cb3d64bSFrederic Barrat * can allocate a pro-rated number of actags to each function. 2692cb3d64bSFrederic Barrat */ 2702cb3d64bSFrederic Barrat if (!link->assignment_done) 2712cb3d64bSFrederic Barrat assign_actags(link); 2722cb3d64bSFrederic Barrat 2732cb3d64bSFrederic Barrat *base = link->fn_actags[PCI_FUNC(dev->devfn)].start; 2742cb3d64bSFrederic Barrat *enabled = link->fn_actags[PCI_FUNC(dev->devfn)].count; 2752cb3d64bSFrederic Barrat *supported = link->fn_desired_actags[PCI_FUNC(dev->devfn)]; 2762cb3d64bSFrederic Barrat 2772cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 2782cb3d64bSFrederic Barrat return 0; 2792cb3d64bSFrederic Barrat } 2802cb3d64bSFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_actag); 2812cb3d64bSFrederic Barrat 2822cb3d64bSFrederic Barrat int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count) 2832cb3d64bSFrederic Barrat { 2842cb3d64bSFrederic Barrat struct npu_link *link; 2852cb3d64bSFrederic Barrat int i, rc = -EINVAL; 2862cb3d64bSFrederic Barrat 2872cb3d64bSFrederic Barrat /* 2882cb3d64bSFrederic Barrat * The number of PASIDs (process address space ID) which can 2892cb3d64bSFrederic Barrat * be used by a function depends on how many functions exist 2902cb3d64bSFrederic Barrat * on the device. The NPU needs to be configured to know how 2912cb3d64bSFrederic Barrat * many bits are available to PASIDs and how many are to be 2921fd02f66SJulia Lawall * used by the function BDF identifier. 2932cb3d64bSFrederic Barrat * 2942cb3d64bSFrederic Barrat * We only support one AFU-carrying function for now. 2952cb3d64bSFrederic Barrat */ 2962cb3d64bSFrederic Barrat mutex_lock(&links_list_lock); 2972cb3d64bSFrederic Barrat 2982cb3d64bSFrederic Barrat link = find_link(dev); 2992cb3d64bSFrederic Barrat if (!link) { 3002cb3d64bSFrederic Barrat dev_err(&dev->dev, "actag information not found\n"); 3012cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 3022cb3d64bSFrederic Barrat return -ENODEV; 3032cb3d64bSFrederic Barrat } 3042cb3d64bSFrederic Barrat 3052cb3d64bSFrederic Barrat for (i = 0; i < 8; i++) 3062cb3d64bSFrederic Barrat if (link->fn_desired_actags[i] && (i == PCI_FUNC(dev->devfn))) { 3072cb3d64bSFrederic Barrat *count = PNV_OCXL_PASID_MAX; 3082cb3d64bSFrederic Barrat rc = 0; 3092cb3d64bSFrederic Barrat break; 3102cb3d64bSFrederic Barrat } 3112cb3d64bSFrederic Barrat 3122cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock); 3132cb3d64bSFrederic Barrat dev_dbg(&dev->dev, "%d PASIDs available for function\n", 3142cb3d64bSFrederic Barrat rc ? 0 : *count); 3152cb3d64bSFrederic Barrat return rc; 3162cb3d64bSFrederic Barrat } 3172cb3d64bSFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_pasid_count); 3186914c757SFrederic Barrat 3196914c757SFrederic Barrat static void set_templ_rate(unsigned int templ, unsigned int rate, char *buf) 3206914c757SFrederic Barrat { 3216914c757SFrederic Barrat int shift, idx; 3226914c757SFrederic Barrat 3236914c757SFrederic Barrat WARN_ON(templ > PNV_OCXL_TL_MAX_TEMPLATE); 3246914c757SFrederic Barrat idx = (PNV_OCXL_TL_MAX_TEMPLATE - templ) / 2; 3256914c757SFrederic Barrat shift = 4 * (1 - ((PNV_OCXL_TL_MAX_TEMPLATE - templ) % 2)); 3266914c757SFrederic Barrat buf[idx] |= rate << shift; 3276914c757SFrederic Barrat } 3286914c757SFrederic Barrat 3296914c757SFrederic Barrat int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, 3306914c757SFrederic Barrat char *rate_buf, int rate_buf_size) 3316914c757SFrederic Barrat { 3326914c757SFrederic Barrat if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE) 3336914c757SFrederic Barrat return -EINVAL; 3346914c757SFrederic Barrat /* 3356914c757SFrederic Barrat * The TL capabilities are a characteristic of the NPU, so 3366914c757SFrederic Barrat * we go with hard-coded values. 3376914c757SFrederic Barrat * 3386914c757SFrederic Barrat * The receiving rate of each template is encoded on 4 bits. 3396914c757SFrederic Barrat * 3406914c757SFrederic Barrat * On P9: 3416914c757SFrederic Barrat * - templates 0 -> 3 are supported 3426914c757SFrederic Barrat * - templates 0, 1 and 3 have a 0 receiving rate 3436914c757SFrederic Barrat * - template 2 has receiving rate of 1 (extra cycle) 3446914c757SFrederic Barrat */ 3456914c757SFrederic Barrat memset(rate_buf, 0, rate_buf_size); 3466914c757SFrederic Barrat set_templ_rate(2, 1, rate_buf); 3476914c757SFrederic Barrat *cap = PNV_OCXL_TL_P9_RECV_CAP; 3486914c757SFrederic Barrat return 0; 3496914c757SFrederic Barrat } 3506914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_tl_cap); 3516914c757SFrederic Barrat 3526914c757SFrederic Barrat int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, 3536914c757SFrederic Barrat uint64_t rate_buf_phys, int rate_buf_size) 3546914c757SFrederic Barrat { 3556914c757SFrederic Barrat struct pci_controller *hose = pci_bus_to_host(dev->bus); 3566914c757SFrederic Barrat struct pnv_phb *phb = hose->private_data; 3576914c757SFrederic Barrat int rc; 3586914c757SFrederic Barrat 3596914c757SFrederic Barrat if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE) 3606914c757SFrederic Barrat return -EINVAL; 3616914c757SFrederic Barrat 3626914c757SFrederic Barrat rc = opal_npu_tl_set(phb->opal_id, dev->devfn, cap, 3636914c757SFrederic Barrat rate_buf_phys, rate_buf_size); 3646914c757SFrederic Barrat if (rc) { 3656914c757SFrederic Barrat dev_err(&dev->dev, "Can't configure host TL: %d\n", rc); 3666914c757SFrederic Barrat return -EINVAL; 3676914c757SFrederic Barrat } 3686914c757SFrederic Barrat return 0; 3696914c757SFrederic Barrat } 3706914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_set_tl_conf); 3716914c757SFrederic Barrat 3726914c757SFrederic Barrat int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq) 3736914c757SFrederic Barrat { 3746914c757SFrederic Barrat int rc; 3756914c757SFrederic Barrat 3766914c757SFrederic Barrat rc = of_property_read_u32(dev->dev.of_node, "ibm,opal-xsl-irq", hwirq); 3776914c757SFrederic Barrat if (rc) { 3786914c757SFrederic Barrat dev_err(&dev->dev, 3796914c757SFrederic Barrat "Can't get translation interrupt for device\n"); 3806914c757SFrederic Barrat return rc; 3816914c757SFrederic Barrat } 3826914c757SFrederic Barrat return 0; 3836914c757SFrederic Barrat } 3846914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_xsl_irq); 3856914c757SFrederic Barrat 3866914c757SFrederic Barrat void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, 3876914c757SFrederic Barrat void __iomem *tfc, void __iomem *pe_handle) 3886914c757SFrederic Barrat { 3896914c757SFrederic Barrat iounmap(dsisr); 3906914c757SFrederic Barrat iounmap(dar); 3916914c757SFrederic Barrat iounmap(tfc); 3926914c757SFrederic Barrat iounmap(pe_handle); 3936914c757SFrederic Barrat } 3946914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_xsl_regs); 3956914c757SFrederic Barrat 3966914c757SFrederic Barrat int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, 3976914c757SFrederic Barrat void __iomem **dar, void __iomem **tfc, 3986914c757SFrederic Barrat void __iomem **pe_handle) 3996914c757SFrederic Barrat { 4006914c757SFrederic Barrat u64 reg; 4016914c757SFrederic Barrat int i, j, rc = 0; 4026914c757SFrederic Barrat void __iomem *regs[4]; 4036914c757SFrederic Barrat 4046914c757SFrederic Barrat /* 4056914c757SFrederic Barrat * opal stores the mmio addresses of the DSISR, DAR, TFC and 4066914c757SFrederic Barrat * PE_HANDLE registers in a device tree property, in that 4076914c757SFrederic Barrat * order 4086914c757SFrederic Barrat */ 4096914c757SFrederic Barrat for (i = 0; i < 4; i++) { 4106914c757SFrederic Barrat rc = of_property_read_u64_index(dev->dev.of_node, 4116914c757SFrederic Barrat "ibm,opal-xsl-mmio", i, ®); 4126914c757SFrederic Barrat if (rc) 4136914c757SFrederic Barrat break; 4146914c757SFrederic Barrat regs[i] = ioremap(reg, 8); 4156914c757SFrederic Barrat if (!regs[i]) { 4166914c757SFrederic Barrat rc = -EINVAL; 4176914c757SFrederic Barrat break; 4186914c757SFrederic Barrat } 4196914c757SFrederic Barrat } 4206914c757SFrederic Barrat if (rc) { 4216914c757SFrederic Barrat dev_err(&dev->dev, "Can't map translation mmio registers\n"); 4226914c757SFrederic Barrat for (j = i - 1; j >= 0; j--) 4236914c757SFrederic Barrat iounmap(regs[j]); 4246914c757SFrederic Barrat } else { 4256914c757SFrederic Barrat *dsisr = regs[0]; 4266914c757SFrederic Barrat *dar = regs[1]; 4276914c757SFrederic Barrat *tfc = regs[2]; 4286914c757SFrederic Barrat *pe_handle = regs[3]; 4296914c757SFrederic Barrat } 4306914c757SFrederic Barrat return rc; 4316914c757SFrederic Barrat } 4326914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_map_xsl_regs); 4336914c757SFrederic Barrat 4346914c757SFrederic Barrat struct spa_data { 4356914c757SFrederic Barrat u64 phb_opal_id; 4366914c757SFrederic Barrat u32 bdfn; 4376914c757SFrederic Barrat }; 4386914c757SFrederic Barrat 4396914c757SFrederic Barrat int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, 4406914c757SFrederic Barrat void **platform_data) 4416914c757SFrederic Barrat { 4426914c757SFrederic Barrat struct pci_controller *hose = pci_bus_to_host(dev->bus); 4436914c757SFrederic Barrat struct pnv_phb *phb = hose->private_data; 4446914c757SFrederic Barrat struct spa_data *data; 4456914c757SFrederic Barrat u32 bdfn; 4466914c757SFrederic Barrat int rc; 4476914c757SFrederic Barrat 4486914c757SFrederic Barrat data = kzalloc(sizeof(*data), GFP_KERNEL); 4496914c757SFrederic Barrat if (!data) 4506914c757SFrederic Barrat return -ENOMEM; 4516914c757SFrederic Barrat 4526914c757SFrederic Barrat bdfn = (dev->bus->number << 8) | dev->devfn; 4536914c757SFrederic Barrat rc = opal_npu_spa_setup(phb->opal_id, bdfn, virt_to_phys(spa_mem), 4546914c757SFrederic Barrat PE_mask); 4556914c757SFrederic Barrat if (rc) { 4566914c757SFrederic Barrat dev_err(&dev->dev, "Can't setup Shared Process Area: %d\n", rc); 4576914c757SFrederic Barrat kfree(data); 4586914c757SFrederic Barrat return rc; 4596914c757SFrederic Barrat } 4606914c757SFrederic Barrat data->phb_opal_id = phb->opal_id; 4616914c757SFrederic Barrat data->bdfn = bdfn; 4626914c757SFrederic Barrat *platform_data = (void *) data; 4636914c757SFrederic Barrat return 0; 4646914c757SFrederic Barrat } 4656914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_spa_setup); 4666914c757SFrederic Barrat 4676914c757SFrederic Barrat void pnv_ocxl_spa_release(void *platform_data) 4686914c757SFrederic Barrat { 4696914c757SFrederic Barrat struct spa_data *data = (struct spa_data *) platform_data; 4706914c757SFrederic Barrat int rc; 4716914c757SFrederic Barrat 4726914c757SFrederic Barrat rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0); 4736914c757SFrederic Barrat WARN_ON(rc); 4746914c757SFrederic Barrat kfree(data); 4756914c757SFrederic Barrat } 4766914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release); 4776914c757SFrederic Barrat 47819df3958SAlastair D'Silva int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle) 4796914c757SFrederic Barrat { 4806914c757SFrederic Barrat struct spa_data *data = (struct spa_data *) platform_data; 4816914c757SFrederic Barrat 482*5e495265Sye xingchen return opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle); 4836914c757SFrederic Barrat } 48419df3958SAlastair D'Silva EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe_from_cache); 485fc1347b5SChristophe Lombard 486fc1347b5SChristophe Lombard int pnv_ocxl_map_lpar(struct pci_dev *dev, uint64_t lparid, 487fc1347b5SChristophe Lombard uint64_t lpcr, void __iomem **arva) 488fc1347b5SChristophe Lombard { 489fc1347b5SChristophe Lombard struct pci_controller *hose = pci_bus_to_host(dev->bus); 490fc1347b5SChristophe Lombard struct pnv_phb *phb = hose->private_data; 491fc1347b5SChristophe Lombard u64 mmio_atsd; 492fc1347b5SChristophe Lombard int rc; 493fc1347b5SChristophe Lombard 494fc1347b5SChristophe Lombard /* ATSD physical address. 495fc1347b5SChristophe Lombard * ATSD LAUNCH register: write access initiates a shoot down to 496fc1347b5SChristophe Lombard * initiate the TLB Invalidate command. 497fc1347b5SChristophe Lombard */ 498fc1347b5SChristophe Lombard rc = of_property_read_u64_index(hose->dn, "ibm,mmio-atsd", 499fc1347b5SChristophe Lombard 0, &mmio_atsd); 500fc1347b5SChristophe Lombard if (rc) { 501fc1347b5SChristophe Lombard dev_info(&dev->dev, "No available ATSD found\n"); 502fc1347b5SChristophe Lombard return rc; 503fc1347b5SChristophe Lombard } 504fc1347b5SChristophe Lombard 505fc1347b5SChristophe Lombard /* Assign a register set to a Logical Partition and MMIO ATSD 506fc1347b5SChristophe Lombard * LPARID register to the required value. 507fc1347b5SChristophe Lombard */ 508fc1347b5SChristophe Lombard rc = opal_npu_map_lpar(phb->opal_id, pci_dev_id(dev), 509fc1347b5SChristophe Lombard lparid, lpcr); 510fc1347b5SChristophe Lombard if (rc) { 511fc1347b5SChristophe Lombard dev_err(&dev->dev, "Error mapping device to LPAR: %d\n", rc); 512fc1347b5SChristophe Lombard return rc; 513fc1347b5SChristophe Lombard } 514fc1347b5SChristophe Lombard 515fc1347b5SChristophe Lombard *arva = ioremap(mmio_atsd, 24); 516fc1347b5SChristophe Lombard if (!(*arva)) { 517fc1347b5SChristophe Lombard dev_warn(&dev->dev, "ioremap failed - mmio_atsd: %#llx\n", mmio_atsd); 518fc1347b5SChristophe Lombard rc = -ENOMEM; 519fc1347b5SChristophe Lombard } 520fc1347b5SChristophe Lombard 521fc1347b5SChristophe Lombard return rc; 522fc1347b5SChristophe Lombard } 523fc1347b5SChristophe Lombard EXPORT_SYMBOL_GPL(pnv_ocxl_map_lpar); 524fc1347b5SChristophe Lombard 525fc1347b5SChristophe Lombard void pnv_ocxl_unmap_lpar(void __iomem *arva) 526fc1347b5SChristophe Lombard { 527fc1347b5SChristophe Lombard iounmap(arva); 528fc1347b5SChristophe Lombard } 529fc1347b5SChristophe Lombard EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_lpar); 53019b311caSChristophe Lombard 53119b311caSChristophe Lombard void pnv_ocxl_tlb_invalidate(void __iomem *arva, 53219b311caSChristophe Lombard unsigned long pid, 53319b311caSChristophe Lombard unsigned long addr, 53419b311caSChristophe Lombard unsigned long page_size) 53519b311caSChristophe Lombard { 53619b311caSChristophe Lombard unsigned long timeout = jiffies + (HZ * PNV_OCXL_ATSD_TIMEOUT); 53719b311caSChristophe Lombard u64 val = 0ull; 53819b311caSChristophe Lombard int pend; 53919b311caSChristophe Lombard u8 size; 54019b311caSChristophe Lombard 54119b311caSChristophe Lombard if (!(arva)) 54219b311caSChristophe Lombard return; 54319b311caSChristophe Lombard 54419b311caSChristophe Lombard if (addr) { 54519b311caSChristophe Lombard /* load Abbreviated Virtual Address register with 54619b311caSChristophe Lombard * the necessary value 54719b311caSChristophe Lombard */ 54819b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_AVA_AVA, addr >> (63-51)); 54919b311caSChristophe Lombard out_be64(arva + PNV_OCXL_ATSD_AVA, val); 55019b311caSChristophe Lombard } 55119b311caSChristophe Lombard 55219b311caSChristophe Lombard /* Write access initiates a shoot down to initiate the 55319b311caSChristophe Lombard * TLB Invalidate command 55419b311caSChristophe Lombard */ 55519b311caSChristophe Lombard val = PNV_OCXL_ATSD_LNCH_R; 55619b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_RIC, 0b10); 55719b311caSChristophe Lombard if (addr) 55819b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_IS, 0b00); 55919b311caSChristophe Lombard else { 56019b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_IS, 0b01); 56119b311caSChristophe Lombard val |= PNV_OCXL_ATSD_LNCH_OCAPI_SINGLETON; 56219b311caSChristophe Lombard } 56319b311caSChristophe Lombard val |= PNV_OCXL_ATSD_LNCH_PRS; 56419b311caSChristophe Lombard /* Actual Page Size to be invalidated 56519b311caSChristophe Lombard * 000 4KB 56619b311caSChristophe Lombard * 101 64KB 56719b311caSChristophe Lombard * 001 2MB 56819b311caSChristophe Lombard * 010 1GB 56919b311caSChristophe Lombard */ 57019b311caSChristophe Lombard size = 0b101; 57119b311caSChristophe Lombard if (page_size == 0x1000) 57219b311caSChristophe Lombard size = 0b000; 57319b311caSChristophe Lombard if (page_size == 0x200000) 57419b311caSChristophe Lombard size = 0b001; 57519b311caSChristophe Lombard if (page_size == 0x40000000) 57619b311caSChristophe Lombard size = 0b010; 57719b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_AP, size); 57819b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_PID, pid); 57919b311caSChristophe Lombard out_be64(arva + PNV_OCXL_ATSD_LNCH, val); 58019b311caSChristophe Lombard 58119b311caSChristophe Lombard /* Poll the ATSD status register to determine when the 58219b311caSChristophe Lombard * TLB Invalidate has been completed. 58319b311caSChristophe Lombard */ 58419b311caSChristophe Lombard val = in_be64(arva + PNV_OCXL_ATSD_STAT); 58519b311caSChristophe Lombard pend = val >> 63; 58619b311caSChristophe Lombard 58719b311caSChristophe Lombard while (pend) { 58819b311caSChristophe Lombard if (time_after_eq(jiffies, timeout)) { 58919b311caSChristophe Lombard pr_err("%s - Timeout while reading XTS MMIO ATSD status register (val=%#llx, pidr=0x%lx)\n", 59019b311caSChristophe Lombard __func__, val, pid); 59119b311caSChristophe Lombard return; 59219b311caSChristophe Lombard } 59319b311caSChristophe Lombard cpu_relax(); 59419b311caSChristophe Lombard val = in_be64(arva + PNV_OCXL_ATSD_STAT); 59519b311caSChristophe Lombard pend = val >> 63; 59619b311caSChristophe Lombard } 59719b311caSChristophe Lombard } 59819b311caSChristophe Lombard EXPORT_SYMBOL_GPL(pnv_ocxl_tlb_invalidate); 599