16914c757SFrederic Barrat // SPDX-License-Identifier: GPL-2.0+
26914c757SFrederic Barrat // Copyright 2017 IBM Corp.
36914c757SFrederic Barrat #include <asm/pnv-ocxl.h>
46914c757SFrederic Barrat #include <asm/opal.h>
52cb3d64bSFrederic Barrat #include <misc/ocxl-config.h>
66914c757SFrederic Barrat #include "pci.h"
76914c757SFrederic Barrat
86914c757SFrederic Barrat #define PNV_OCXL_TL_P9_RECV_CAP 0x000000000000000Full
92cb3d64bSFrederic Barrat #define PNV_OCXL_ACTAG_MAX 64
106914c757SFrederic Barrat /* PASIDs are 20-bit, but on P9, NPU can only handle 15 bits */
116914c757SFrederic Barrat #define PNV_OCXL_PASID_BITS 15
126914c757SFrederic Barrat #define PNV_OCXL_PASID_MAX ((1 << PNV_OCXL_PASID_BITS) - 1)
136914c757SFrederic Barrat
142cb3d64bSFrederic Barrat #define AFU_PRESENT (1 << 31)
152cb3d64bSFrederic Barrat #define AFU_INDEX_MASK 0x3F000000
162cb3d64bSFrederic Barrat #define AFU_INDEX_SHIFT 24
172cb3d64bSFrederic Barrat #define ACTAG_MASK 0xFFF
182cb3d64bSFrederic Barrat
192cb3d64bSFrederic Barrat
202cb3d64bSFrederic Barrat struct actag_range {
212cb3d64bSFrederic Barrat u16 start;
222cb3d64bSFrederic Barrat u16 count;
232cb3d64bSFrederic Barrat };
242cb3d64bSFrederic Barrat
252cb3d64bSFrederic Barrat struct npu_link {
262cb3d64bSFrederic Barrat struct list_head list;
272cb3d64bSFrederic Barrat int domain;
282cb3d64bSFrederic Barrat int bus;
292cb3d64bSFrederic Barrat int dev;
302cb3d64bSFrederic Barrat u16 fn_desired_actags[8];
312cb3d64bSFrederic Barrat struct actag_range fn_actags[8];
322cb3d64bSFrederic Barrat bool assignment_done;
332cb3d64bSFrederic Barrat };
342cb3d64bSFrederic Barrat static struct list_head links_list = LIST_HEAD_INIT(links_list);
352cb3d64bSFrederic Barrat static DEFINE_MUTEX(links_list_lock);
362cb3d64bSFrederic Barrat
372cb3d64bSFrederic Barrat
382cb3d64bSFrederic Barrat /*
392cb3d64bSFrederic Barrat * opencapi actags handling:
402cb3d64bSFrederic Barrat *
412cb3d64bSFrederic Barrat * When sending commands, the opencapi device references the memory
422cb3d64bSFrederic Barrat * context it's targeting with an 'actag', which is really an alias
432cb3d64bSFrederic Barrat * for a (BDF, pasid) combination. When it receives a command, the NPU
442cb3d64bSFrederic Barrat * must do a lookup of the actag to identify the memory context. The
452cb3d64bSFrederic Barrat * hardware supports a finite number of actags per link (64 for
462cb3d64bSFrederic Barrat * POWER9).
472cb3d64bSFrederic Barrat *
482cb3d64bSFrederic Barrat * The device can carry multiple functions, and each function can have
492cb3d64bSFrederic Barrat * multiple AFUs. Each AFU advertises in its config space the number
502cb3d64bSFrederic Barrat * of desired actags. The host must configure in the config space of
512cb3d64bSFrederic Barrat * the AFU how many actags the AFU is really allowed to use (which can
522cb3d64bSFrederic Barrat * be less than what the AFU desires).
532cb3d64bSFrederic Barrat *
542cb3d64bSFrederic Barrat * When a PCI function is probed by the driver, it has no visibility
552cb3d64bSFrederic Barrat * about the other PCI functions and how many actags they'd like,
562cb3d64bSFrederic Barrat * which makes it impossible to distribute actags fairly among AFUs.
572cb3d64bSFrederic Barrat *
582cb3d64bSFrederic Barrat * Unfortunately, the only way to know how many actags a function
592cb3d64bSFrederic Barrat * desires is by looking at the data for each AFU in the config space
602cb3d64bSFrederic Barrat * and add them up. Similarly, the only way to know how many actags
612cb3d64bSFrederic Barrat * all the functions of the physical device desire is by adding the
622cb3d64bSFrederic Barrat * previously computed function counts. Then we can match that against
632cb3d64bSFrederic Barrat * what the hardware supports.
642cb3d64bSFrederic Barrat *
652cb3d64bSFrederic Barrat * To get a comprehensive view, we use a 'pci fixup': at the end of
662cb3d64bSFrederic Barrat * PCI enumeration, each function counts how many actags its AFUs
672cb3d64bSFrederic Barrat * desire and we save it in a 'npu_link' structure, shared between all
682cb3d64bSFrederic Barrat * the PCI functions of a same device. Therefore, when the first
692cb3d64bSFrederic Barrat * function is probed by the driver, we can get an idea of the total
702cb3d64bSFrederic Barrat * count of desired actags for the device, and assign the actags to
712cb3d64bSFrederic Barrat * the AFUs, by pro-rating if needed.
722cb3d64bSFrederic Barrat */
732cb3d64bSFrederic Barrat
find_dvsec_from_pos(struct pci_dev * dev,int dvsec_id,int pos)742cb3d64bSFrederic Barrat static int find_dvsec_from_pos(struct pci_dev *dev, int dvsec_id, int pos)
752cb3d64bSFrederic Barrat {
762cb3d64bSFrederic Barrat int vsec = pos;
772cb3d64bSFrederic Barrat u16 vendor, id;
782cb3d64bSFrederic Barrat
792cb3d64bSFrederic Barrat while ((vsec = pci_find_next_ext_capability(dev, vsec,
802cb3d64bSFrederic Barrat OCXL_EXT_CAP_ID_DVSEC))) {
812cb3d64bSFrederic Barrat pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
822cb3d64bSFrederic Barrat &vendor);
832cb3d64bSFrederic Barrat pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
842cb3d64bSFrederic Barrat if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id)
852cb3d64bSFrederic Barrat return vsec;
862cb3d64bSFrederic Barrat }
872cb3d64bSFrederic Barrat return 0;
882cb3d64bSFrederic Barrat }
892cb3d64bSFrederic Barrat
find_dvsec_afu_ctrl(struct pci_dev * dev,u8 afu_idx)902cb3d64bSFrederic Barrat static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)
912cb3d64bSFrederic Barrat {
922cb3d64bSFrederic Barrat int vsec = 0;
932cb3d64bSFrederic Barrat u8 idx;
942cb3d64bSFrederic Barrat
952cb3d64bSFrederic Barrat while ((vsec = find_dvsec_from_pos(dev, OCXL_DVSEC_AFU_CTRL_ID,
962cb3d64bSFrederic Barrat vsec))) {
972cb3d64bSFrederic Barrat pci_read_config_byte(dev, vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX,
982cb3d64bSFrederic Barrat &idx);
992cb3d64bSFrederic Barrat if (idx == afu_idx)
1002cb3d64bSFrederic Barrat return vsec;
1012cb3d64bSFrederic Barrat }
1022cb3d64bSFrederic Barrat return 0;
1032cb3d64bSFrederic Barrat }
1042cb3d64bSFrederic Barrat
get_max_afu_index(struct pci_dev * dev,int * afu_idx)1052cb3d64bSFrederic Barrat static int get_max_afu_index(struct pci_dev *dev, int *afu_idx)
1062cb3d64bSFrederic Barrat {
1072cb3d64bSFrederic Barrat int pos;
1082cb3d64bSFrederic Barrat u32 val;
1092cb3d64bSFrederic Barrat
110c6d7e134SBen Widawsky pos = pci_find_dvsec_capability(dev, PCI_VENDOR_ID_IBM,
111c6d7e134SBen Widawsky OCXL_DVSEC_FUNC_ID);
1122cb3d64bSFrederic Barrat if (!pos)
1132cb3d64bSFrederic Barrat return -ESRCH;
1142cb3d64bSFrederic Barrat
1152cb3d64bSFrederic Barrat pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val);
1162cb3d64bSFrederic Barrat if (val & AFU_PRESENT)
1172cb3d64bSFrederic Barrat *afu_idx = (val & AFU_INDEX_MASK) >> AFU_INDEX_SHIFT;
1182cb3d64bSFrederic Barrat else
1192cb3d64bSFrederic Barrat *afu_idx = -1;
1202cb3d64bSFrederic Barrat return 0;
1212cb3d64bSFrederic Barrat }
1222cb3d64bSFrederic Barrat
get_actag_count(struct pci_dev * dev,int afu_idx,int * actag)1232cb3d64bSFrederic Barrat static int get_actag_count(struct pci_dev *dev, int afu_idx, int *actag)
1242cb3d64bSFrederic Barrat {
1252cb3d64bSFrederic Barrat int pos;
1262cb3d64bSFrederic Barrat u16 actag_sup;
1272cb3d64bSFrederic Barrat
1282cb3d64bSFrederic Barrat pos = find_dvsec_afu_ctrl(dev, afu_idx);
1292cb3d64bSFrederic Barrat if (!pos)
1302cb3d64bSFrederic Barrat return -ESRCH;
1312cb3d64bSFrederic Barrat
1322cb3d64bSFrederic Barrat pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP,
1332cb3d64bSFrederic Barrat &actag_sup);
1342cb3d64bSFrederic Barrat *actag = actag_sup & ACTAG_MASK;
1352cb3d64bSFrederic Barrat return 0;
1362cb3d64bSFrederic Barrat }
1372cb3d64bSFrederic Barrat
find_link(struct pci_dev * dev)1382cb3d64bSFrederic Barrat static struct npu_link *find_link(struct pci_dev *dev)
1392cb3d64bSFrederic Barrat {
1402cb3d64bSFrederic Barrat struct npu_link *link;
1412cb3d64bSFrederic Barrat
1422cb3d64bSFrederic Barrat list_for_each_entry(link, &links_list, list) {
1432cb3d64bSFrederic Barrat /* The functions of a device all share the same link */
1442cb3d64bSFrederic Barrat if (link->domain == pci_domain_nr(dev->bus) &&
1452cb3d64bSFrederic Barrat link->bus == dev->bus->number &&
1462cb3d64bSFrederic Barrat link->dev == PCI_SLOT(dev->devfn)) {
1472cb3d64bSFrederic Barrat return link;
1482cb3d64bSFrederic Barrat }
1492cb3d64bSFrederic Barrat }
1502cb3d64bSFrederic Barrat
1512cb3d64bSFrederic Barrat /* link doesn't exist yet. Allocate one */
1522cb3d64bSFrederic Barrat link = kzalloc(sizeof(struct npu_link), GFP_KERNEL);
1532cb3d64bSFrederic Barrat if (!link)
1542cb3d64bSFrederic Barrat return NULL;
1552cb3d64bSFrederic Barrat link->domain = pci_domain_nr(dev->bus);
1562cb3d64bSFrederic Barrat link->bus = dev->bus->number;
1572cb3d64bSFrederic Barrat link->dev = PCI_SLOT(dev->devfn);
1582cb3d64bSFrederic Barrat list_add(&link->list, &links_list);
1592cb3d64bSFrederic Barrat return link;
1602cb3d64bSFrederic Barrat }
1612cb3d64bSFrederic Barrat
pnv_ocxl_fixup_actag(struct pci_dev * dev)1622cb3d64bSFrederic Barrat static void pnv_ocxl_fixup_actag(struct pci_dev *dev)
1632cb3d64bSFrederic Barrat {
1642cb3d64bSFrederic Barrat struct pci_controller *hose = pci_bus_to_host(dev->bus);
1652cb3d64bSFrederic Barrat struct pnv_phb *phb = hose->private_data;
1662cb3d64bSFrederic Barrat struct npu_link *link;
1672cb3d64bSFrederic Barrat int rc, afu_idx = -1, i, actag;
1682cb3d64bSFrederic Barrat
1692cb3d64bSFrederic Barrat if (!machine_is(powernv))
1702cb3d64bSFrederic Barrat return;
1712cb3d64bSFrederic Barrat
1722cb3d64bSFrederic Barrat if (phb->type != PNV_PHB_NPU_OCAPI)
1732cb3d64bSFrederic Barrat return;
1742cb3d64bSFrederic Barrat
1752cb3d64bSFrederic Barrat mutex_lock(&links_list_lock);
1762cb3d64bSFrederic Barrat
1772cb3d64bSFrederic Barrat link = find_link(dev);
1782cb3d64bSFrederic Barrat if (!link) {
1792cb3d64bSFrederic Barrat dev_warn(&dev->dev, "couldn't update actag information\n");
1802cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock);
1812cb3d64bSFrederic Barrat return;
1822cb3d64bSFrederic Barrat }
1832cb3d64bSFrederic Barrat
1842cb3d64bSFrederic Barrat /*
1852cb3d64bSFrederic Barrat * Check how many actags are desired for the AFUs under that
1862cb3d64bSFrederic Barrat * function and add it to the count for the link
1872cb3d64bSFrederic Barrat */
1882cb3d64bSFrederic Barrat rc = get_max_afu_index(dev, &afu_idx);
1892cb3d64bSFrederic Barrat if (rc) {
1902cb3d64bSFrederic Barrat /* Most likely an invalid config space */
1912cb3d64bSFrederic Barrat dev_dbg(&dev->dev, "couldn't find AFU information\n");
1922cb3d64bSFrederic Barrat afu_idx = -1;
1932cb3d64bSFrederic Barrat }
1942cb3d64bSFrederic Barrat
1952cb3d64bSFrederic Barrat link->fn_desired_actags[PCI_FUNC(dev->devfn)] = 0;
1962cb3d64bSFrederic Barrat for (i = 0; i <= afu_idx; i++) {
1972cb3d64bSFrederic Barrat /*
1982cb3d64bSFrederic Barrat * AFU index 'holes' are allowed. So don't fail if we
1992cb3d64bSFrederic Barrat * can't read the actag info for an index
2002cb3d64bSFrederic Barrat */
2012cb3d64bSFrederic Barrat rc = get_actag_count(dev, i, &actag);
2022cb3d64bSFrederic Barrat if (rc)
2032cb3d64bSFrederic Barrat continue;
2042cb3d64bSFrederic Barrat link->fn_desired_actags[PCI_FUNC(dev->devfn)] += actag;
2052cb3d64bSFrederic Barrat }
2062cb3d64bSFrederic Barrat dev_dbg(&dev->dev, "total actags for function: %d\n",
2072cb3d64bSFrederic Barrat link->fn_desired_actags[PCI_FUNC(dev->devfn)]);
2082cb3d64bSFrederic Barrat
2092cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock);
2102cb3d64bSFrederic Barrat }
2112cb3d64bSFrederic Barrat DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pnv_ocxl_fixup_actag);
2122cb3d64bSFrederic Barrat
assign_fn_actags(u16 desired,u16 total)2132cb3d64bSFrederic Barrat static u16 assign_fn_actags(u16 desired, u16 total)
2142cb3d64bSFrederic Barrat {
2152cb3d64bSFrederic Barrat u16 count;
2162cb3d64bSFrederic Barrat
2172cb3d64bSFrederic Barrat if (total <= PNV_OCXL_ACTAG_MAX)
2182cb3d64bSFrederic Barrat count = desired;
2192cb3d64bSFrederic Barrat else
2202cb3d64bSFrederic Barrat count = PNV_OCXL_ACTAG_MAX * desired / total;
2212cb3d64bSFrederic Barrat
2222cb3d64bSFrederic Barrat return count;
2232cb3d64bSFrederic Barrat }
2242cb3d64bSFrederic Barrat
assign_actags(struct npu_link * link)2252cb3d64bSFrederic Barrat static void assign_actags(struct npu_link *link)
2262cb3d64bSFrederic Barrat {
2272cb3d64bSFrederic Barrat u16 actag_count, range_start = 0, total_desired = 0;
2282cb3d64bSFrederic Barrat int i;
2292cb3d64bSFrederic Barrat
2302cb3d64bSFrederic Barrat for (i = 0; i < 8; i++)
2312cb3d64bSFrederic Barrat total_desired += link->fn_desired_actags[i];
2322cb3d64bSFrederic Barrat
2332cb3d64bSFrederic Barrat for (i = 0; i < 8; i++) {
2342cb3d64bSFrederic Barrat if (link->fn_desired_actags[i]) {
2352cb3d64bSFrederic Barrat actag_count = assign_fn_actags(
2362cb3d64bSFrederic Barrat link->fn_desired_actags[i],
2372cb3d64bSFrederic Barrat total_desired);
2382cb3d64bSFrederic Barrat link->fn_actags[i].start = range_start;
2392cb3d64bSFrederic Barrat link->fn_actags[i].count = actag_count;
2402cb3d64bSFrederic Barrat range_start += actag_count;
2412cb3d64bSFrederic Barrat WARN_ON(range_start >= PNV_OCXL_ACTAG_MAX);
2422cb3d64bSFrederic Barrat }
2432cb3d64bSFrederic Barrat pr_debug("link %x:%x:%x fct %d actags: start=%d count=%d (desired=%d)\n",
2442cb3d64bSFrederic Barrat link->domain, link->bus, link->dev, i,
2452cb3d64bSFrederic Barrat link->fn_actags[i].start, link->fn_actags[i].count,
2462cb3d64bSFrederic Barrat link->fn_desired_actags[i]);
2472cb3d64bSFrederic Barrat }
2482cb3d64bSFrederic Barrat link->assignment_done = true;
2492cb3d64bSFrederic Barrat }
2502cb3d64bSFrederic Barrat
pnv_ocxl_get_actag(struct pci_dev * dev,u16 * base,u16 * enabled,u16 * supported)2512cb3d64bSFrederic Barrat int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled,
2522cb3d64bSFrederic Barrat u16 *supported)
2532cb3d64bSFrederic Barrat {
2542cb3d64bSFrederic Barrat struct npu_link *link;
2552cb3d64bSFrederic Barrat
2562cb3d64bSFrederic Barrat mutex_lock(&links_list_lock);
2572cb3d64bSFrederic Barrat
2582cb3d64bSFrederic Barrat link = find_link(dev);
2592cb3d64bSFrederic Barrat if (!link) {
2602cb3d64bSFrederic Barrat dev_err(&dev->dev, "actag information not found\n");
2612cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock);
2622cb3d64bSFrederic Barrat return -ENODEV;
2632cb3d64bSFrederic Barrat }
2642cb3d64bSFrederic Barrat /*
2652cb3d64bSFrederic Barrat * On p9, we only have 64 actags per link, so they must be
2662cb3d64bSFrederic Barrat * shared by all the functions of the same adapter. We counted
2672cb3d64bSFrederic Barrat * the desired actag counts during PCI enumeration, so that we
2682cb3d64bSFrederic Barrat * can allocate a pro-rated number of actags to each function.
2692cb3d64bSFrederic Barrat */
2702cb3d64bSFrederic Barrat if (!link->assignment_done)
2712cb3d64bSFrederic Barrat assign_actags(link);
2722cb3d64bSFrederic Barrat
2732cb3d64bSFrederic Barrat *base = link->fn_actags[PCI_FUNC(dev->devfn)].start;
2742cb3d64bSFrederic Barrat *enabled = link->fn_actags[PCI_FUNC(dev->devfn)].count;
2752cb3d64bSFrederic Barrat *supported = link->fn_desired_actags[PCI_FUNC(dev->devfn)];
2762cb3d64bSFrederic Barrat
2772cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock);
2782cb3d64bSFrederic Barrat return 0;
2792cb3d64bSFrederic Barrat }
2802cb3d64bSFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_actag);
2812cb3d64bSFrederic Barrat
pnv_ocxl_get_pasid_count(struct pci_dev * dev,int * count)2822cb3d64bSFrederic Barrat int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count)
2832cb3d64bSFrederic Barrat {
2842cb3d64bSFrederic Barrat struct npu_link *link;
2852cb3d64bSFrederic Barrat int i, rc = -EINVAL;
2862cb3d64bSFrederic Barrat
2872cb3d64bSFrederic Barrat /*
2882cb3d64bSFrederic Barrat * The number of PASIDs (process address space ID) which can
2892cb3d64bSFrederic Barrat * be used by a function depends on how many functions exist
2902cb3d64bSFrederic Barrat * on the device. The NPU needs to be configured to know how
2912cb3d64bSFrederic Barrat * many bits are available to PASIDs and how many are to be
2921fd02f66SJulia Lawall * used by the function BDF identifier.
2932cb3d64bSFrederic Barrat *
2942cb3d64bSFrederic Barrat * We only support one AFU-carrying function for now.
2952cb3d64bSFrederic Barrat */
2962cb3d64bSFrederic Barrat mutex_lock(&links_list_lock);
2972cb3d64bSFrederic Barrat
2982cb3d64bSFrederic Barrat link = find_link(dev);
2992cb3d64bSFrederic Barrat if (!link) {
3002cb3d64bSFrederic Barrat dev_err(&dev->dev, "actag information not found\n");
3012cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock);
3022cb3d64bSFrederic Barrat return -ENODEV;
3032cb3d64bSFrederic Barrat }
3042cb3d64bSFrederic Barrat
3052cb3d64bSFrederic Barrat for (i = 0; i < 8; i++)
3062cb3d64bSFrederic Barrat if (link->fn_desired_actags[i] && (i == PCI_FUNC(dev->devfn))) {
3072cb3d64bSFrederic Barrat *count = PNV_OCXL_PASID_MAX;
3082cb3d64bSFrederic Barrat rc = 0;
3092cb3d64bSFrederic Barrat break;
3102cb3d64bSFrederic Barrat }
3112cb3d64bSFrederic Barrat
3122cb3d64bSFrederic Barrat mutex_unlock(&links_list_lock);
3132cb3d64bSFrederic Barrat dev_dbg(&dev->dev, "%d PASIDs available for function\n",
3142cb3d64bSFrederic Barrat rc ? 0 : *count);
3152cb3d64bSFrederic Barrat return rc;
3162cb3d64bSFrederic Barrat }
3172cb3d64bSFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_pasid_count);
3186914c757SFrederic Barrat
set_templ_rate(unsigned int templ,unsigned int rate,char * buf)3196914c757SFrederic Barrat static void set_templ_rate(unsigned int templ, unsigned int rate, char *buf)
3206914c757SFrederic Barrat {
3216914c757SFrederic Barrat int shift, idx;
3226914c757SFrederic Barrat
3236914c757SFrederic Barrat WARN_ON(templ > PNV_OCXL_TL_MAX_TEMPLATE);
3246914c757SFrederic Barrat idx = (PNV_OCXL_TL_MAX_TEMPLATE - templ) / 2;
3256914c757SFrederic Barrat shift = 4 * (1 - ((PNV_OCXL_TL_MAX_TEMPLATE - templ) % 2));
3266914c757SFrederic Barrat buf[idx] |= rate << shift;
3276914c757SFrederic Barrat }
3286914c757SFrederic Barrat
pnv_ocxl_get_tl_cap(struct pci_dev * dev,long * cap,char * rate_buf,int rate_buf_size)3296914c757SFrederic Barrat int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap,
3306914c757SFrederic Barrat char *rate_buf, int rate_buf_size)
3316914c757SFrederic Barrat {
3326914c757SFrederic Barrat if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE)
3336914c757SFrederic Barrat return -EINVAL;
3346914c757SFrederic Barrat /*
3356914c757SFrederic Barrat * The TL capabilities are a characteristic of the NPU, so
3366914c757SFrederic Barrat * we go with hard-coded values.
3376914c757SFrederic Barrat *
3386914c757SFrederic Barrat * The receiving rate of each template is encoded on 4 bits.
3396914c757SFrederic Barrat *
3406914c757SFrederic Barrat * On P9:
3416914c757SFrederic Barrat * - templates 0 -> 3 are supported
3426914c757SFrederic Barrat * - templates 0, 1 and 3 have a 0 receiving rate
3436914c757SFrederic Barrat * - template 2 has receiving rate of 1 (extra cycle)
3446914c757SFrederic Barrat */
3456914c757SFrederic Barrat memset(rate_buf, 0, rate_buf_size);
3466914c757SFrederic Barrat set_templ_rate(2, 1, rate_buf);
3476914c757SFrederic Barrat *cap = PNV_OCXL_TL_P9_RECV_CAP;
3486914c757SFrederic Barrat return 0;
3496914c757SFrederic Barrat }
3506914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_tl_cap);
3516914c757SFrederic Barrat
pnv_ocxl_set_tl_conf(struct pci_dev * dev,long cap,uint64_t rate_buf_phys,int rate_buf_size)3526914c757SFrederic Barrat int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap,
3536914c757SFrederic Barrat uint64_t rate_buf_phys, int rate_buf_size)
3546914c757SFrederic Barrat {
3556914c757SFrederic Barrat struct pci_controller *hose = pci_bus_to_host(dev->bus);
3566914c757SFrederic Barrat struct pnv_phb *phb = hose->private_data;
3576914c757SFrederic Barrat int rc;
3586914c757SFrederic Barrat
3596914c757SFrederic Barrat if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE)
3606914c757SFrederic Barrat return -EINVAL;
3616914c757SFrederic Barrat
3626914c757SFrederic Barrat rc = opal_npu_tl_set(phb->opal_id, dev->devfn, cap,
3636914c757SFrederic Barrat rate_buf_phys, rate_buf_size);
3646914c757SFrederic Barrat if (rc) {
3656914c757SFrederic Barrat dev_err(&dev->dev, "Can't configure host TL: %d\n", rc);
3666914c757SFrederic Barrat return -EINVAL;
3676914c757SFrederic Barrat }
3686914c757SFrederic Barrat return 0;
3696914c757SFrederic Barrat }
3706914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_set_tl_conf);
3716914c757SFrederic Barrat
pnv_ocxl_get_xsl_irq(struct pci_dev * dev,int * hwirq)3726914c757SFrederic Barrat int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq)
3736914c757SFrederic Barrat {
3746914c757SFrederic Barrat int rc;
3756914c757SFrederic Barrat
3766914c757SFrederic Barrat rc = of_property_read_u32(dev->dev.of_node, "ibm,opal-xsl-irq", hwirq);
3776914c757SFrederic Barrat if (rc) {
3786914c757SFrederic Barrat dev_err(&dev->dev,
3796914c757SFrederic Barrat "Can't get translation interrupt for device\n");
3806914c757SFrederic Barrat return rc;
3816914c757SFrederic Barrat }
3826914c757SFrederic Barrat return 0;
3836914c757SFrederic Barrat }
3846914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_get_xsl_irq);
3856914c757SFrederic Barrat
pnv_ocxl_unmap_xsl_regs(void __iomem * dsisr,void __iomem * dar,void __iomem * tfc,void __iomem * pe_handle)3866914c757SFrederic Barrat void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar,
3876914c757SFrederic Barrat void __iomem *tfc, void __iomem *pe_handle)
3886914c757SFrederic Barrat {
3896914c757SFrederic Barrat iounmap(dsisr);
3906914c757SFrederic Barrat iounmap(dar);
3916914c757SFrederic Barrat iounmap(tfc);
3926914c757SFrederic Barrat iounmap(pe_handle);
3936914c757SFrederic Barrat }
3946914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_xsl_regs);
3956914c757SFrederic Barrat
pnv_ocxl_map_xsl_regs(struct pci_dev * dev,void __iomem ** dsisr,void __iomem ** dar,void __iomem ** tfc,void __iomem ** pe_handle)3966914c757SFrederic Barrat int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr,
3976914c757SFrederic Barrat void __iomem **dar, void __iomem **tfc,
3986914c757SFrederic Barrat void __iomem **pe_handle)
3996914c757SFrederic Barrat {
4006914c757SFrederic Barrat u64 reg;
4016914c757SFrederic Barrat int i, j, rc = 0;
4026914c757SFrederic Barrat void __iomem *regs[4];
4036914c757SFrederic Barrat
4046914c757SFrederic Barrat /*
4056914c757SFrederic Barrat * opal stores the mmio addresses of the DSISR, DAR, TFC and
4066914c757SFrederic Barrat * PE_HANDLE registers in a device tree property, in that
4076914c757SFrederic Barrat * order
4086914c757SFrederic Barrat */
4096914c757SFrederic Barrat for (i = 0; i < 4; i++) {
4106914c757SFrederic Barrat rc = of_property_read_u64_index(dev->dev.of_node,
4116914c757SFrederic Barrat "ibm,opal-xsl-mmio", i, ®);
4126914c757SFrederic Barrat if (rc)
4136914c757SFrederic Barrat break;
4146914c757SFrederic Barrat regs[i] = ioremap(reg, 8);
4156914c757SFrederic Barrat if (!regs[i]) {
4166914c757SFrederic Barrat rc = -EINVAL;
4176914c757SFrederic Barrat break;
4186914c757SFrederic Barrat }
4196914c757SFrederic Barrat }
4206914c757SFrederic Barrat if (rc) {
4216914c757SFrederic Barrat dev_err(&dev->dev, "Can't map translation mmio registers\n");
4226914c757SFrederic Barrat for (j = i - 1; j >= 0; j--)
4236914c757SFrederic Barrat iounmap(regs[j]);
4246914c757SFrederic Barrat } else {
4256914c757SFrederic Barrat *dsisr = regs[0];
4266914c757SFrederic Barrat *dar = regs[1];
4276914c757SFrederic Barrat *tfc = regs[2];
4286914c757SFrederic Barrat *pe_handle = regs[3];
4296914c757SFrederic Barrat }
4306914c757SFrederic Barrat return rc;
4316914c757SFrederic Barrat }
4326914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_map_xsl_regs);
4336914c757SFrederic Barrat
4346914c757SFrederic Barrat struct spa_data {
4356914c757SFrederic Barrat u64 phb_opal_id;
4366914c757SFrederic Barrat u32 bdfn;
4376914c757SFrederic Barrat };
4386914c757SFrederic Barrat
pnv_ocxl_spa_setup(struct pci_dev * dev,void * spa_mem,int PE_mask,void ** platform_data)4396914c757SFrederic Barrat int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask,
4406914c757SFrederic Barrat void **platform_data)
4416914c757SFrederic Barrat {
4426914c757SFrederic Barrat struct pci_controller *hose = pci_bus_to_host(dev->bus);
4436914c757SFrederic Barrat struct pnv_phb *phb = hose->private_data;
4446914c757SFrederic Barrat struct spa_data *data;
4456914c757SFrederic Barrat u32 bdfn;
4466914c757SFrederic Barrat int rc;
4476914c757SFrederic Barrat
4486914c757SFrederic Barrat data = kzalloc(sizeof(*data), GFP_KERNEL);
4496914c757SFrederic Barrat if (!data)
4506914c757SFrederic Barrat return -ENOMEM;
4516914c757SFrederic Barrat
452*075a88d5SZheng Zengkai bdfn = pci_dev_id(dev);
4536914c757SFrederic Barrat rc = opal_npu_spa_setup(phb->opal_id, bdfn, virt_to_phys(spa_mem),
4546914c757SFrederic Barrat PE_mask);
4556914c757SFrederic Barrat if (rc) {
4566914c757SFrederic Barrat dev_err(&dev->dev, "Can't setup Shared Process Area: %d\n", rc);
4576914c757SFrederic Barrat kfree(data);
4586914c757SFrederic Barrat return rc;
4596914c757SFrederic Barrat }
4606914c757SFrederic Barrat data->phb_opal_id = phb->opal_id;
4616914c757SFrederic Barrat data->bdfn = bdfn;
4626914c757SFrederic Barrat *platform_data = (void *) data;
4636914c757SFrederic Barrat return 0;
4646914c757SFrederic Barrat }
4656914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_spa_setup);
4666914c757SFrederic Barrat
pnv_ocxl_spa_release(void * platform_data)4676914c757SFrederic Barrat void pnv_ocxl_spa_release(void *platform_data)
4686914c757SFrederic Barrat {
4696914c757SFrederic Barrat struct spa_data *data = (struct spa_data *) platform_data;
4706914c757SFrederic Barrat int rc;
4716914c757SFrederic Barrat
4726914c757SFrederic Barrat rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0);
4736914c757SFrederic Barrat WARN_ON(rc);
4746914c757SFrederic Barrat kfree(data);
4756914c757SFrederic Barrat }
4766914c757SFrederic Barrat EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release);
4776914c757SFrederic Barrat
pnv_ocxl_spa_remove_pe_from_cache(void * platform_data,int pe_handle)47819df3958SAlastair D'Silva int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle)
4796914c757SFrederic Barrat {
4806914c757SFrederic Barrat struct spa_data *data = (struct spa_data *) platform_data;
4816914c757SFrederic Barrat
4825e495265Sye xingchen return opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle);
4836914c757SFrederic Barrat }
48419df3958SAlastair D'Silva EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe_from_cache);
485fc1347b5SChristophe Lombard
pnv_ocxl_map_lpar(struct pci_dev * dev,uint64_t lparid,uint64_t lpcr,void __iomem ** arva)486fc1347b5SChristophe Lombard int pnv_ocxl_map_lpar(struct pci_dev *dev, uint64_t lparid,
487fc1347b5SChristophe Lombard uint64_t lpcr, void __iomem **arva)
488fc1347b5SChristophe Lombard {
489fc1347b5SChristophe Lombard struct pci_controller *hose = pci_bus_to_host(dev->bus);
490fc1347b5SChristophe Lombard struct pnv_phb *phb = hose->private_data;
491fc1347b5SChristophe Lombard u64 mmio_atsd;
492fc1347b5SChristophe Lombard int rc;
493fc1347b5SChristophe Lombard
494fc1347b5SChristophe Lombard /* ATSD physical address.
495fc1347b5SChristophe Lombard * ATSD LAUNCH register: write access initiates a shoot down to
496fc1347b5SChristophe Lombard * initiate the TLB Invalidate command.
497fc1347b5SChristophe Lombard */
498fc1347b5SChristophe Lombard rc = of_property_read_u64_index(hose->dn, "ibm,mmio-atsd",
499fc1347b5SChristophe Lombard 0, &mmio_atsd);
500fc1347b5SChristophe Lombard if (rc) {
501fc1347b5SChristophe Lombard dev_info(&dev->dev, "No available ATSD found\n");
502fc1347b5SChristophe Lombard return rc;
503fc1347b5SChristophe Lombard }
504fc1347b5SChristophe Lombard
505fc1347b5SChristophe Lombard /* Assign a register set to a Logical Partition and MMIO ATSD
506fc1347b5SChristophe Lombard * LPARID register to the required value.
507fc1347b5SChristophe Lombard */
508fc1347b5SChristophe Lombard rc = opal_npu_map_lpar(phb->opal_id, pci_dev_id(dev),
509fc1347b5SChristophe Lombard lparid, lpcr);
510fc1347b5SChristophe Lombard if (rc) {
511fc1347b5SChristophe Lombard dev_err(&dev->dev, "Error mapping device to LPAR: %d\n", rc);
512fc1347b5SChristophe Lombard return rc;
513fc1347b5SChristophe Lombard }
514fc1347b5SChristophe Lombard
515fc1347b5SChristophe Lombard *arva = ioremap(mmio_atsd, 24);
516fc1347b5SChristophe Lombard if (!(*arva)) {
517fc1347b5SChristophe Lombard dev_warn(&dev->dev, "ioremap failed - mmio_atsd: %#llx\n", mmio_atsd);
518fc1347b5SChristophe Lombard rc = -ENOMEM;
519fc1347b5SChristophe Lombard }
520fc1347b5SChristophe Lombard
521fc1347b5SChristophe Lombard return rc;
522fc1347b5SChristophe Lombard }
523fc1347b5SChristophe Lombard EXPORT_SYMBOL_GPL(pnv_ocxl_map_lpar);
524fc1347b5SChristophe Lombard
pnv_ocxl_unmap_lpar(void __iomem * arva)525fc1347b5SChristophe Lombard void pnv_ocxl_unmap_lpar(void __iomem *arva)
526fc1347b5SChristophe Lombard {
527fc1347b5SChristophe Lombard iounmap(arva);
528fc1347b5SChristophe Lombard }
529fc1347b5SChristophe Lombard EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_lpar);
53019b311caSChristophe Lombard
pnv_ocxl_tlb_invalidate(void __iomem * arva,unsigned long pid,unsigned long addr,unsigned long page_size)53119b311caSChristophe Lombard void pnv_ocxl_tlb_invalidate(void __iomem *arva,
53219b311caSChristophe Lombard unsigned long pid,
53319b311caSChristophe Lombard unsigned long addr,
53419b311caSChristophe Lombard unsigned long page_size)
53519b311caSChristophe Lombard {
53619b311caSChristophe Lombard unsigned long timeout = jiffies + (HZ * PNV_OCXL_ATSD_TIMEOUT);
53719b311caSChristophe Lombard u64 val = 0ull;
53819b311caSChristophe Lombard int pend;
53919b311caSChristophe Lombard u8 size;
54019b311caSChristophe Lombard
54119b311caSChristophe Lombard if (!(arva))
54219b311caSChristophe Lombard return;
54319b311caSChristophe Lombard
54419b311caSChristophe Lombard if (addr) {
54519b311caSChristophe Lombard /* load Abbreviated Virtual Address register with
54619b311caSChristophe Lombard * the necessary value
54719b311caSChristophe Lombard */
54819b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_AVA_AVA, addr >> (63-51));
54919b311caSChristophe Lombard out_be64(arva + PNV_OCXL_ATSD_AVA, val);
55019b311caSChristophe Lombard }
55119b311caSChristophe Lombard
55219b311caSChristophe Lombard /* Write access initiates a shoot down to initiate the
55319b311caSChristophe Lombard * TLB Invalidate command
55419b311caSChristophe Lombard */
55519b311caSChristophe Lombard val = PNV_OCXL_ATSD_LNCH_R;
55619b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_RIC, 0b10);
55719b311caSChristophe Lombard if (addr)
55819b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_IS, 0b00);
55919b311caSChristophe Lombard else {
56019b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_IS, 0b01);
56119b311caSChristophe Lombard val |= PNV_OCXL_ATSD_LNCH_OCAPI_SINGLETON;
56219b311caSChristophe Lombard }
56319b311caSChristophe Lombard val |= PNV_OCXL_ATSD_LNCH_PRS;
56419b311caSChristophe Lombard /* Actual Page Size to be invalidated
56519b311caSChristophe Lombard * 000 4KB
56619b311caSChristophe Lombard * 101 64KB
56719b311caSChristophe Lombard * 001 2MB
56819b311caSChristophe Lombard * 010 1GB
56919b311caSChristophe Lombard */
57019b311caSChristophe Lombard size = 0b101;
57119b311caSChristophe Lombard if (page_size == 0x1000)
57219b311caSChristophe Lombard size = 0b000;
57319b311caSChristophe Lombard if (page_size == 0x200000)
57419b311caSChristophe Lombard size = 0b001;
57519b311caSChristophe Lombard if (page_size == 0x40000000)
57619b311caSChristophe Lombard size = 0b010;
57719b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_AP, size);
57819b311caSChristophe Lombard val |= FIELD_PREP(PNV_OCXL_ATSD_LNCH_PID, pid);
57919b311caSChristophe Lombard out_be64(arva + PNV_OCXL_ATSD_LNCH, val);
58019b311caSChristophe Lombard
58119b311caSChristophe Lombard /* Poll the ATSD status register to determine when the
58219b311caSChristophe Lombard * TLB Invalidate has been completed.
58319b311caSChristophe Lombard */
58419b311caSChristophe Lombard val = in_be64(arva + PNV_OCXL_ATSD_STAT);
58519b311caSChristophe Lombard pend = val >> 63;
58619b311caSChristophe Lombard
58719b311caSChristophe Lombard while (pend) {
58819b311caSChristophe Lombard if (time_after_eq(jiffies, timeout)) {
58919b311caSChristophe Lombard pr_err("%s - Timeout while reading XTS MMIO ATSD status register (val=%#llx, pidr=0x%lx)\n",
59019b311caSChristophe Lombard __func__, val, pid);
59119b311caSChristophe Lombard return;
59219b311caSChristophe Lombard }
59319b311caSChristophe Lombard cpu_relax();
59419b311caSChristophe Lombard val = in_be64(arva + PNV_OCXL_ATSD_STAT);
59519b311caSChristophe Lombard pend = val >> 63;
59619b311caSChristophe Lombard }
59719b311caSChristophe Lombard }
59819b311caSChristophe Lombard EXPORT_SYMBOL_GPL(pnv_ocxl_tlb_invalidate);
599