12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds ** IA64 System Bus Adapter (SBA) I/O MMU manager
41da177e4SLinus Torvalds **
55f6602a1SAlex Williamson ** (c) Copyright 2002-2005 Alex Williamson
61da177e4SLinus Torvalds ** (c) Copyright 2002-2003 Grant Grundler
75f6602a1SAlex Williamson ** (c) Copyright 2002-2005 Hewlett-Packard Company
81da177e4SLinus Torvalds **
91da177e4SLinus Torvalds ** Portions (c) 2000 Grant Grundler (from parisc I/O MMU code)
101da177e4SLinus Torvalds ** Portions (c) 1999 Dave S. Miller (from sparc64 I/O MMU code)
111da177e4SLinus Torvalds **
121da177e4SLinus Torvalds **
131da177e4SLinus Torvalds **
141da177e4SLinus Torvalds ** This module initializes the IOC (I/O Controller) found on HP
151da177e4SLinus Torvalds ** McKinley machines and their successors.
161da177e4SLinus Torvalds **
171da177e4SLinus Torvalds */
181da177e4SLinus Torvalds
191da177e4SLinus Torvalds #include <linux/types.h>
201da177e4SLinus Torvalds #include <linux/kernel.h>
211da177e4SLinus Torvalds #include <linux/module.h>
221da177e4SLinus Torvalds #include <linux/spinlock.h>
231da177e4SLinus Torvalds #include <linux/slab.h>
241da177e4SLinus Torvalds #include <linux/init.h>
251da177e4SLinus Torvalds #include <linux/mm.h>
261da177e4SLinus Torvalds #include <linux/string.h>
271da177e4SLinus Torvalds #include <linux/pci.h>
281da177e4SLinus Torvalds #include <linux/proc_fs.h>
291da177e4SLinus Torvalds #include <linux/seq_file.h>
301da177e4SLinus Torvalds #include <linux/acpi.h>
311da177e4SLinus Torvalds #include <linux/efi.h>
321da177e4SLinus Torvalds #include <linux/nodemask.h>
331da177e4SLinus Torvalds #include <linux/bitops.h> /* hweight64() */
3451b58e3eSTerry Loftin #include <linux/crash_dump.h>
35b34eb53cSFUJITA Tomonori #include <linux/iommu-helper.h>
360a0f0d8bSChristoph Hellwig #include <linux/dma-map-ops.h>
37268bb0ceSLinus Torvalds #include <linux/prefetch.h>
38974f83ecSChristoph Hellwig #include <linux/swiotlb.h>
391da177e4SLinus Torvalds
401da177e4SLinus Torvalds #include <asm/delay.h> /* ia64_get_itc() */
411da177e4SLinus Torvalds #include <asm/io.h>
421da177e4SLinus Torvalds #include <asm/page.h> /* PAGE_OFFSET */
431da177e4SLinus Torvalds #include <asm/dma.h>
441da177e4SLinus Torvalds
451da177e4SLinus Torvalds #include <asm/acpi-ext.h>
461da177e4SLinus Torvalds
471da177e4SLinus Torvalds #define PFX "IOC: "
481da177e4SLinus Torvalds
491da177e4SLinus Torvalds /*
501da177e4SLinus Torvalds ** Enabling timing search of the pdir resource map. Output in /proc.
511da177e4SLinus Torvalds ** Disabled by default to optimize performance.
521da177e4SLinus Torvalds */
531da177e4SLinus Torvalds #undef PDIR_SEARCH_TIMING
541da177e4SLinus Torvalds
551da177e4SLinus Torvalds /*
561da177e4SLinus Torvalds ** This option allows cards capable of 64bit DMA to bypass the IOMMU. If
571da177e4SLinus Torvalds ** not defined, all DMA will be 32bit and go through the TLB.
581da177e4SLinus Torvalds ** There's potentially a conflict in the bio merge code with us
591da177e4SLinus Torvalds ** advertising an iommu, but then bypassing it. Since I/O MMU bypassing
601da177e4SLinus Torvalds ** appears to give more performance than bio-level virtual merging, we'll
611da177e4SLinus Torvalds ** do the former for now. NOTE: BYPASS_SG also needs to be undef'd to
621da177e4SLinus Torvalds ** completely restrict DMA to the IOMMU.
631da177e4SLinus Torvalds */
641da177e4SLinus Torvalds #define ALLOW_IOV_BYPASS
651da177e4SLinus Torvalds
661da177e4SLinus Torvalds /*
671da177e4SLinus Torvalds ** This option specifically allows/disallows bypassing scatterlists with
681da177e4SLinus Torvalds ** multiple entries. Coalescing these entries can allow better DMA streaming
691da177e4SLinus Torvalds ** and in some cases shows better performance than entirely bypassing the
701da177e4SLinus Torvalds ** IOMMU. Performance increase on the order of 1-2% sequential output/input
711da177e4SLinus Torvalds ** using bonnie++ on a RAID0 MD device (sym2 & mpt).
721da177e4SLinus Torvalds */
731da177e4SLinus Torvalds #undef ALLOW_IOV_BYPASS_SG
741da177e4SLinus Torvalds
751da177e4SLinus Torvalds /*
761da177e4SLinus Torvalds ** If a device prefetches beyond the end of a valid pdir entry, it will cause
771da177e4SLinus Torvalds ** a hard failure, ie. MCA. Version 3.0 and later of the zx1 LBA should
781da177e4SLinus Torvalds ** disconnect on 4k boundaries and prevent such issues. If the device is
790779bf2dSMatt LaPlante ** particularly aggressive, this option will keep the entire pdir valid such
801da177e4SLinus Torvalds ** that prefetching will hit a valid address. This could severely impact
811da177e4SLinus Torvalds ** error containment, and is therefore off by default. The page that is
821da177e4SLinus Torvalds ** used for spill-over is poisoned, so that should help debugging somewhat.
831da177e4SLinus Torvalds */
841da177e4SLinus Torvalds #undef FULL_VALID_PDIR
851da177e4SLinus Torvalds
861da177e4SLinus Torvalds #define ENABLE_MARK_CLEAN
871da177e4SLinus Torvalds
881da177e4SLinus Torvalds /*
891da177e4SLinus Torvalds ** The number of debug flags is a clue - this code is fragile. NOTE: since
901da177e4SLinus Torvalds ** tightening the use of res_lock the resource bitmap and actual pdir are no
911da177e4SLinus Torvalds ** longer guaranteed to stay in sync. The sanity checking code isn't going to
921da177e4SLinus Torvalds ** like that.
931da177e4SLinus Torvalds */
941da177e4SLinus Torvalds #undef DEBUG_SBA_INIT
951da177e4SLinus Torvalds #undef DEBUG_SBA_RUN
961da177e4SLinus Torvalds #undef DEBUG_SBA_RUN_SG
971da177e4SLinus Torvalds #undef DEBUG_SBA_RESOURCE
981da177e4SLinus Torvalds #undef ASSERT_PDIR_SANITY
991da177e4SLinus Torvalds #undef DEBUG_LARGE_SG_ENTRIES
1001da177e4SLinus Torvalds #undef DEBUG_BYPASS
1011da177e4SLinus Torvalds
1021da177e4SLinus Torvalds #if defined(FULL_VALID_PDIR) && defined(ASSERT_PDIR_SANITY)
1031da177e4SLinus Torvalds #error FULL_VALID_PDIR and ASSERT_PDIR_SANITY are mutually exclusive
1041da177e4SLinus Torvalds #endif
1051da177e4SLinus Torvalds
1061da177e4SLinus Torvalds #define SBA_INLINE __inline__
1071da177e4SLinus Torvalds /* #define SBA_INLINE */
1081da177e4SLinus Torvalds
1091da177e4SLinus Torvalds #ifdef DEBUG_SBA_INIT
1101da177e4SLinus Torvalds #define DBG_INIT(x...) printk(x)
1111da177e4SLinus Torvalds #else
1121da177e4SLinus Torvalds #define DBG_INIT(x...)
1131da177e4SLinus Torvalds #endif
1141da177e4SLinus Torvalds
1151da177e4SLinus Torvalds #ifdef DEBUG_SBA_RUN
1161da177e4SLinus Torvalds #define DBG_RUN(x...) printk(x)
1171da177e4SLinus Torvalds #else
1181da177e4SLinus Torvalds #define DBG_RUN(x...)
1191da177e4SLinus Torvalds #endif
1201da177e4SLinus Torvalds
1211da177e4SLinus Torvalds #ifdef DEBUG_SBA_RUN_SG
1221da177e4SLinus Torvalds #define DBG_RUN_SG(x...) printk(x)
1231da177e4SLinus Torvalds #else
1241da177e4SLinus Torvalds #define DBG_RUN_SG(x...)
1251da177e4SLinus Torvalds #endif
1261da177e4SLinus Torvalds
1271da177e4SLinus Torvalds
1281da177e4SLinus Torvalds #ifdef DEBUG_SBA_RESOURCE
1291da177e4SLinus Torvalds #define DBG_RES(x...) printk(x)
1301da177e4SLinus Torvalds #else
1311da177e4SLinus Torvalds #define DBG_RES(x...)
1321da177e4SLinus Torvalds #endif
1331da177e4SLinus Torvalds
1341da177e4SLinus Torvalds #ifdef DEBUG_BYPASS
1351da177e4SLinus Torvalds #define DBG_BYPASS(x...) printk(x)
1361da177e4SLinus Torvalds #else
1371da177e4SLinus Torvalds #define DBG_BYPASS(x...)
1381da177e4SLinus Torvalds #endif
1391da177e4SLinus Torvalds
1401da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
1411da177e4SLinus Torvalds #define ASSERT(expr) \
1421da177e4SLinus Torvalds if(!(expr)) { \
1431da177e4SLinus Torvalds printk( "\n" __FILE__ ":%d: Assertion " #expr " failed!\n",__LINE__); \
1441da177e4SLinus Torvalds panic(#expr); \
1451da177e4SLinus Torvalds }
1461da177e4SLinus Torvalds #else
1471da177e4SLinus Torvalds #define ASSERT(expr)
1481da177e4SLinus Torvalds #endif
1491da177e4SLinus Torvalds
1501da177e4SLinus Torvalds /*
1511da177e4SLinus Torvalds ** The number of pdir entries to "free" before issuing
1521da177e4SLinus Torvalds ** a read to PCOM register to flush out PCOM writes.
1531da177e4SLinus Torvalds ** Interacts with allocation granularity (ie 4 or 8 entries
1541da177e4SLinus Torvalds ** allocated and free'd/purged at a time might make this
1551da177e4SLinus Torvalds ** less interesting).
1561da177e4SLinus Torvalds */
1571da177e4SLinus Torvalds #define DELAYED_RESOURCE_CNT 64
1581da177e4SLinus Torvalds
159e15da401SBjorn Helgaas #define PCI_DEVICE_ID_HP_SX2000_IOC 0x12ec
160e15da401SBjorn Helgaas
1611da177e4SLinus Torvalds #define ZX1_IOC_ID ((PCI_DEVICE_ID_HP_ZX1_IOC << 16) | PCI_VENDOR_ID_HP)
1621da177e4SLinus Torvalds #define ZX2_IOC_ID ((PCI_DEVICE_ID_HP_ZX2_IOC << 16) | PCI_VENDOR_ID_HP)
1631da177e4SLinus Torvalds #define REO_IOC_ID ((PCI_DEVICE_ID_HP_REO_IOC << 16) | PCI_VENDOR_ID_HP)
1641da177e4SLinus Torvalds #define SX1000_IOC_ID ((PCI_DEVICE_ID_HP_SX1000_IOC << 16) | PCI_VENDOR_ID_HP)
165e15da401SBjorn Helgaas #define SX2000_IOC_ID ((PCI_DEVICE_ID_HP_SX2000_IOC << 16) | PCI_VENDOR_ID_HP)
1661da177e4SLinus Torvalds
1671da177e4SLinus Torvalds #define ZX1_IOC_OFFSET 0x1000 /* ACPI reports SBA, we want IOC */
1681da177e4SLinus Torvalds
1691da177e4SLinus Torvalds #define IOC_FUNC_ID 0x000
1701da177e4SLinus Torvalds #define IOC_FCLASS 0x008 /* function class, bist, header, rev... */
1711da177e4SLinus Torvalds #define IOC_IBASE 0x300 /* IO TLB */
1721da177e4SLinus Torvalds #define IOC_IMASK 0x308
1731da177e4SLinus Torvalds #define IOC_PCOM 0x310
1741da177e4SLinus Torvalds #define IOC_TCNFG 0x318
1751da177e4SLinus Torvalds #define IOC_PDIR_BASE 0x320
1761da177e4SLinus Torvalds
1771da177e4SLinus Torvalds #define IOC_ROPE0_CFG 0x500
1781da177e4SLinus Torvalds #define IOC_ROPE_AO 0x10 /* Allow "Relaxed Ordering" */
1791da177e4SLinus Torvalds
1801da177e4SLinus Torvalds
1811da177e4SLinus Torvalds /* AGP GART driver looks for this */
1821da177e4SLinus Torvalds #define ZX1_SBA_IOMMU_COOKIE 0x0000badbadc0ffeeUL
1831da177e4SLinus Torvalds
1841da177e4SLinus Torvalds /*
1851da177e4SLinus Torvalds ** The zx1 IOC supports 4/8/16/64KB page sizes (see TCNFG register)
1861da177e4SLinus Torvalds **
1871da177e4SLinus Torvalds ** Some IOCs (sx1000) can run at the above pages sizes, but are
1881da177e4SLinus Torvalds ** really only supported using the IOC at a 4k page size.
1891da177e4SLinus Torvalds **
1901da177e4SLinus Torvalds ** iovp_size could only be greater than PAGE_SIZE if we are
1911da177e4SLinus Torvalds ** confident the drivers really only touch the next physical
1921da177e4SLinus Torvalds ** page iff that driver instance owns it.
1931da177e4SLinus Torvalds */
1941da177e4SLinus Torvalds static unsigned long iovp_size;
1951da177e4SLinus Torvalds static unsigned long iovp_shift;
1961da177e4SLinus Torvalds static unsigned long iovp_mask;
1971da177e4SLinus Torvalds
1981da177e4SLinus Torvalds struct ioc {
1991da177e4SLinus Torvalds void __iomem *ioc_hpa; /* I/O MMU base address */
2001da177e4SLinus Torvalds char *res_map; /* resource map, bit == pdir entry */
2011da177e4SLinus Torvalds u64 *pdir_base; /* physical base address */
2021da177e4SLinus Torvalds unsigned long ibase; /* pdir IOV Space base */
2031da177e4SLinus Torvalds unsigned long imask; /* pdir IOV Space mask */
2041da177e4SLinus Torvalds
2051da177e4SLinus Torvalds unsigned long *res_hint; /* next avail IOVP - circular search */
2061da177e4SLinus Torvalds unsigned long dma_mask;
2071da177e4SLinus Torvalds spinlock_t res_lock; /* protects the resource bitmap, but must be held when */
2081da177e4SLinus Torvalds /* clearing pdir to prevent races with allocations. */
2091da177e4SLinus Torvalds unsigned int res_bitshift; /* from the RIGHT! */
2101da177e4SLinus Torvalds unsigned int res_size; /* size of resource map in bytes */
2111da177e4SLinus Torvalds #ifdef CONFIG_NUMA
2121da177e4SLinus Torvalds unsigned int node; /* node where this IOC lives */
2131da177e4SLinus Torvalds #endif
2141da177e4SLinus Torvalds #if DELAYED_RESOURCE_CNT > 0
2151da177e4SLinus Torvalds spinlock_t saved_lock; /* may want to try to get this on a separate cacheline */
2161da177e4SLinus Torvalds /* than res_lock for bigger systems. */
2171da177e4SLinus Torvalds int saved_cnt;
2181da177e4SLinus Torvalds struct sba_dma_pair {
2191da177e4SLinus Torvalds dma_addr_t iova;
2201da177e4SLinus Torvalds size_t size;
2211da177e4SLinus Torvalds } saved[DELAYED_RESOURCE_CNT];
2221da177e4SLinus Torvalds #endif
2231da177e4SLinus Torvalds
2241da177e4SLinus Torvalds #ifdef PDIR_SEARCH_TIMING
2251da177e4SLinus Torvalds #define SBA_SEARCH_SAMPLE 0x100
2261da177e4SLinus Torvalds unsigned long avg_search[SBA_SEARCH_SAMPLE];
2271da177e4SLinus Torvalds unsigned long avg_idx; /* current index into avg_search */
2281da177e4SLinus Torvalds #endif
2291da177e4SLinus Torvalds
2301da177e4SLinus Torvalds /* Stuff we don't need in performance path */
2311da177e4SLinus Torvalds struct ioc *next; /* list of IOC's in system */
2321da177e4SLinus Torvalds acpi_handle handle; /* for multiple IOC's */
2331da177e4SLinus Torvalds const char *name;
2341da177e4SLinus Torvalds unsigned int func_id;
2351da177e4SLinus Torvalds unsigned int rev; /* HW revision of chip */
2361da177e4SLinus Torvalds u32 iov_size;
2371da177e4SLinus Torvalds unsigned int pdir_size; /* in bytes, determined by IOV Space size */
2381da177e4SLinus Torvalds struct pci_dev *sac_only_dev;
2391da177e4SLinus Torvalds };
2401da177e4SLinus Torvalds
24112e27b11SRafael J. Wysocki static struct ioc *ioc_list, *ioc_found;
2421da177e4SLinus Torvalds static int reserve_sba_gart = 1;
2431da177e4SLinus Torvalds
2441da177e4SLinus Torvalds static SBA_INLINE void sba_mark_invalid(struct ioc *, dma_addr_t, size_t);
2451da177e4SLinus Torvalds static SBA_INLINE void sba_free_range(struct ioc *, dma_addr_t, size_t);
2461da177e4SLinus Torvalds
24758b053e4SJens Axboe #define sba_sg_address(sg) sg_virt((sg))
2481da177e4SLinus Torvalds
2491da177e4SLinus Torvalds #ifdef FULL_VALID_PDIR
2501da177e4SLinus Torvalds static u64 prefetch_spill_page;
2511da177e4SLinus Torvalds #endif
2521da177e4SLinus Torvalds
253c7797d67SYijing Wang #define GET_IOC(dev) ((dev_is_pci(dev)) \
2541da177e4SLinus Torvalds ? ((struct ioc *) PCI_CONTROLLER(to_pci_dev(dev))->iommu) : NULL)
2551da177e4SLinus Torvalds
2561da177e4SLinus Torvalds /*
2571da177e4SLinus Torvalds ** DMA_CHUNK_SIZE is used by the SCSI mid-layer to break up
2580779bf2dSMatt LaPlante ** (or rather not merge) DMAs into manageable chunks.
2591da177e4SLinus Torvalds ** On parisc, this is more of the software/tuning constraint
2600779bf2dSMatt LaPlante ** rather than the HW. I/O MMU allocation algorithms can be
2610779bf2dSMatt LaPlante ** faster with smaller sizes (to some degree).
2621da177e4SLinus Torvalds */
2631da177e4SLinus Torvalds #define DMA_CHUNK_SIZE (BITS_PER_LONG*iovp_size)
2641da177e4SLinus Torvalds
2651da177e4SLinus Torvalds #define ROUNDUP(x,y) ((x + ((y)-1)) & ~((y)-1))
2661da177e4SLinus Torvalds
2671da177e4SLinus Torvalds /************************************
2681da177e4SLinus Torvalds ** SBA register read and write support
2691da177e4SLinus Torvalds **
2701da177e4SLinus Torvalds ** BE WARNED: register writes are posted.
2711da177e4SLinus Torvalds ** (ie follow writes which must reach HW with a read)
2721da177e4SLinus Torvalds **
2731da177e4SLinus Torvalds */
2741da177e4SLinus Torvalds #define READ_REG(addr) __raw_readq(addr)
2751da177e4SLinus Torvalds #define WRITE_REG(val, addr) __raw_writeq(val, addr)
2761da177e4SLinus Torvalds
2771da177e4SLinus Torvalds #ifdef DEBUG_SBA_INIT
2781da177e4SLinus Torvalds
2791da177e4SLinus Torvalds /**
2801da177e4SLinus Torvalds * sba_dump_tlb - debugging only - print IOMMU operating parameters
2811da177e4SLinus Torvalds * @hpa: base address of the IOMMU
2821da177e4SLinus Torvalds *
2831da177e4SLinus Torvalds * Print the size/location of the IO MMU PDIR.
2841da177e4SLinus Torvalds */
2851da177e4SLinus Torvalds static void
sba_dump_tlb(char * hpa)2861da177e4SLinus Torvalds sba_dump_tlb(char *hpa)
2871da177e4SLinus Torvalds {
2881da177e4SLinus Torvalds DBG_INIT("IO TLB at 0x%p\n", (void *)hpa);
2891da177e4SLinus Torvalds DBG_INIT("IOC_IBASE : %016lx\n", READ_REG(hpa+IOC_IBASE));
2901da177e4SLinus Torvalds DBG_INIT("IOC_IMASK : %016lx\n", READ_REG(hpa+IOC_IMASK));
2911da177e4SLinus Torvalds DBG_INIT("IOC_TCNFG : %016lx\n", READ_REG(hpa+IOC_TCNFG));
2921da177e4SLinus Torvalds DBG_INIT("IOC_PDIR_BASE: %016lx\n", READ_REG(hpa+IOC_PDIR_BASE));
2931da177e4SLinus Torvalds DBG_INIT("\n");
2941da177e4SLinus Torvalds }
2951da177e4SLinus Torvalds #endif
2961da177e4SLinus Torvalds
2971da177e4SLinus Torvalds
2981da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
2991da177e4SLinus Torvalds
3001da177e4SLinus Torvalds /**
3011da177e4SLinus Torvalds * sba_dump_pdir_entry - debugging only - print one IOMMU PDIR entry
3021da177e4SLinus Torvalds * @ioc: IO MMU structure which owns the pdir we are interested in.
3031da177e4SLinus Torvalds * @msg: text to print ont the output line.
3041da177e4SLinus Torvalds * @pide: pdir index.
3051da177e4SLinus Torvalds *
3061da177e4SLinus Torvalds * Print one entry of the IO MMU PDIR in human readable form.
3071da177e4SLinus Torvalds */
3081da177e4SLinus Torvalds static void
sba_dump_pdir_entry(struct ioc * ioc,char * msg,uint pide)3091da177e4SLinus Torvalds sba_dump_pdir_entry(struct ioc *ioc, char *msg, uint pide)
3101da177e4SLinus Torvalds {
3111da177e4SLinus Torvalds /* start printing from lowest pde in rval */
3121da177e4SLinus Torvalds u64 *ptr = &ioc->pdir_base[pide & ~(BITS_PER_LONG - 1)];
3131da177e4SLinus Torvalds unsigned long *rptr = (unsigned long *) &ioc->res_map[(pide >>3) & -sizeof(unsigned long)];
3141da177e4SLinus Torvalds uint rcnt;
3151da177e4SLinus Torvalds
3161da177e4SLinus Torvalds printk(KERN_DEBUG "SBA: %s rp %p bit %d rval 0x%lx\n",
3171da177e4SLinus Torvalds msg, rptr, pide & (BITS_PER_LONG - 1), *rptr);
3181da177e4SLinus Torvalds
3191da177e4SLinus Torvalds rcnt = 0;
3201da177e4SLinus Torvalds while (rcnt < BITS_PER_LONG) {
3211da177e4SLinus Torvalds printk(KERN_DEBUG "%s %2d %p %016Lx\n",
3221da177e4SLinus Torvalds (rcnt == (pide & (BITS_PER_LONG - 1)))
3231da177e4SLinus Torvalds ? " -->" : " ",
3241da177e4SLinus Torvalds rcnt, ptr, (unsigned long long) *ptr );
3251da177e4SLinus Torvalds rcnt++;
3261da177e4SLinus Torvalds ptr++;
3271da177e4SLinus Torvalds }
3281da177e4SLinus Torvalds printk(KERN_DEBUG "%s", msg);
3291da177e4SLinus Torvalds }
3301da177e4SLinus Torvalds
3311da177e4SLinus Torvalds
3321da177e4SLinus Torvalds /**
3331da177e4SLinus Torvalds * sba_check_pdir - debugging only - consistency checker
3341da177e4SLinus Torvalds * @ioc: IO MMU structure which owns the pdir we are interested in.
3351da177e4SLinus Torvalds * @msg: text to print ont the output line.
3361da177e4SLinus Torvalds *
3371da177e4SLinus Torvalds * Verify the resource map and pdir state is consistent
3381da177e4SLinus Torvalds */
3391da177e4SLinus Torvalds static int
sba_check_pdir(struct ioc * ioc,char * msg)3401da177e4SLinus Torvalds sba_check_pdir(struct ioc *ioc, char *msg)
3411da177e4SLinus Torvalds {
3421da177e4SLinus Torvalds u64 *rptr_end = (u64 *) &(ioc->res_map[ioc->res_size]);
3431da177e4SLinus Torvalds u64 *rptr = (u64 *) ioc->res_map; /* resource map ptr */
3441da177e4SLinus Torvalds u64 *pptr = ioc->pdir_base; /* pdir ptr */
3451da177e4SLinus Torvalds uint pide = 0;
3461da177e4SLinus Torvalds
3471da177e4SLinus Torvalds while (rptr < rptr_end) {
3481da177e4SLinus Torvalds u64 rval;
3491da177e4SLinus Torvalds int rcnt; /* number of bits we might check */
3501da177e4SLinus Torvalds
3511da177e4SLinus Torvalds rval = *rptr;
3521da177e4SLinus Torvalds rcnt = 64;
3531da177e4SLinus Torvalds
3541da177e4SLinus Torvalds while (rcnt) {
3551da177e4SLinus Torvalds /* Get last byte and highest bit from that */
3561da177e4SLinus Torvalds u32 pde = ((u32)((*pptr >> (63)) & 0x1));
3571da177e4SLinus Torvalds if ((rval & 0x1) ^ pde)
3581da177e4SLinus Torvalds {
3591da177e4SLinus Torvalds /*
3601da177e4SLinus Torvalds ** BUMMER! -- res_map != pdir --
3611da177e4SLinus Torvalds ** Dump rval and matching pdir entries
3621da177e4SLinus Torvalds */
3631da177e4SLinus Torvalds sba_dump_pdir_entry(ioc, msg, pide);
3641da177e4SLinus Torvalds return(1);
3651da177e4SLinus Torvalds }
3661da177e4SLinus Torvalds rcnt--;
3671da177e4SLinus Torvalds rval >>= 1; /* try the next bit */
3681da177e4SLinus Torvalds pptr++;
3691da177e4SLinus Torvalds pide++;
3701da177e4SLinus Torvalds }
3711da177e4SLinus Torvalds rptr++; /* look at next word of res_map */
3721da177e4SLinus Torvalds }
3731da177e4SLinus Torvalds /* It'd be nice if we always got here :^) */
3741da177e4SLinus Torvalds return 0;
3751da177e4SLinus Torvalds }
3761da177e4SLinus Torvalds
3771da177e4SLinus Torvalds
3781da177e4SLinus Torvalds /**
3791da177e4SLinus Torvalds * sba_dump_sg - debugging only - print Scatter-Gather list
3801da177e4SLinus Torvalds * @ioc: IO MMU structure which owns the pdir we are interested in.
3811da177e4SLinus Torvalds * @startsg: head of the SG list
3821da177e4SLinus Torvalds * @nents: number of entries in SG list
3831da177e4SLinus Torvalds *
3841da177e4SLinus Torvalds * print the SG list so we can verify it's correct by hand.
3851da177e4SLinus Torvalds */
3861da177e4SLinus Torvalds static void
sba_dump_sg(struct ioc * ioc,struct scatterlist * startsg,int nents)3871da177e4SLinus Torvalds sba_dump_sg( struct ioc *ioc, struct scatterlist *startsg, int nents)
3881da177e4SLinus Torvalds {
3891da177e4SLinus Torvalds while (nents-- > 0) {
3901da177e4SLinus Torvalds printk(KERN_DEBUG " %d : DMA %08lx/%05x CPU %p\n", nents,
3911da177e4SLinus Torvalds startsg->dma_address, startsg->dma_length,
3921da177e4SLinus Torvalds sba_sg_address(startsg));
3939b6eccfcSJens Axboe startsg = sg_next(startsg);
3941da177e4SLinus Torvalds }
3951da177e4SLinus Torvalds }
3961da177e4SLinus Torvalds
3971da177e4SLinus Torvalds static void
sba_check_sg(struct ioc * ioc,struct scatterlist * startsg,int nents)3981da177e4SLinus Torvalds sba_check_sg( struct ioc *ioc, struct scatterlist *startsg, int nents)
3991da177e4SLinus Torvalds {
4001da177e4SLinus Torvalds struct scatterlist *the_sg = startsg;
4011da177e4SLinus Torvalds int the_nents = nents;
4021da177e4SLinus Torvalds
4031da177e4SLinus Torvalds while (the_nents-- > 0) {
4041da177e4SLinus Torvalds if (sba_sg_address(the_sg) == 0x0UL)
4051da177e4SLinus Torvalds sba_dump_sg(NULL, startsg, nents);
4069b6eccfcSJens Axboe the_sg = sg_next(the_sg);
4071da177e4SLinus Torvalds }
4081da177e4SLinus Torvalds }
4091da177e4SLinus Torvalds
4101da177e4SLinus Torvalds #endif /* ASSERT_PDIR_SANITY */
4111da177e4SLinus Torvalds
4121da177e4SLinus Torvalds
4131da177e4SLinus Torvalds
4141da177e4SLinus Torvalds
4151da177e4SLinus Torvalds /**************************************************************
4161da177e4SLinus Torvalds *
4171da177e4SLinus Torvalds * I/O Pdir Resource Management
4181da177e4SLinus Torvalds *
4191da177e4SLinus Torvalds * Bits set in the resource map are in use.
4201da177e4SLinus Torvalds * Each bit can represent a number of pages.
4211da177e4SLinus Torvalds * LSbs represent lower addresses (IOVA's).
4221da177e4SLinus Torvalds *
4231da177e4SLinus Torvalds ***************************************************************/
4241da177e4SLinus Torvalds #define PAGES_PER_RANGE 1 /* could increase this to 4 or 8 if needed */
4251da177e4SLinus Torvalds
4261da177e4SLinus Torvalds /* Convert from IOVP to IOVA and vice versa. */
4271da177e4SLinus Torvalds #define SBA_IOVA(ioc,iovp,offset) ((ioc->ibase) | (iovp) | (offset))
4281da177e4SLinus Torvalds #define SBA_IOVP(ioc,iova) ((iova) & ~(ioc->ibase))
4291da177e4SLinus Torvalds
4301da177e4SLinus Torvalds #define PDIR_ENTRY_SIZE sizeof(u64)
4311da177e4SLinus Torvalds
4321da177e4SLinus Torvalds #define PDIR_INDEX(iovp) ((iovp)>>iovp_shift)
4331da177e4SLinus Torvalds
4341da177e4SLinus Torvalds #define RESMAP_MASK(n) ~(~0UL << (n))
4351da177e4SLinus Torvalds #define RESMAP_IDX_MASK (sizeof(unsigned long) - 1)
4361da177e4SLinus Torvalds
4371da177e4SLinus Torvalds
4381da177e4SLinus Torvalds /**
4391da177e4SLinus Torvalds * For most cases the normal get_order is sufficient, however it limits us
4401da177e4SLinus Torvalds * to PAGE_SIZE being the minimum mapping alignment and TC flush granularity.
4411da177e4SLinus Torvalds * It only incurs about 1 clock cycle to use this one with the static variable
4421da177e4SLinus Torvalds * and makes the code more intuitive.
4431da177e4SLinus Torvalds */
4441da177e4SLinus Torvalds static SBA_INLINE int
get_iovp_order(unsigned long size)4451da177e4SLinus Torvalds get_iovp_order (unsigned long size)
4461da177e4SLinus Torvalds {
4471da177e4SLinus Torvalds long double d = size - 1;
4481da177e4SLinus Torvalds long order;
4491da177e4SLinus Torvalds
4501da177e4SLinus Torvalds order = ia64_getf_exp(d);
4511da177e4SLinus Torvalds order = order - iovp_shift - 0xffff + 1;
4521da177e4SLinus Torvalds if (order < 0)
4531da177e4SLinus Torvalds order = 0;
4541da177e4SLinus Torvalds return order;
4551da177e4SLinus Torvalds }
4561da177e4SLinus Torvalds
ptr_to_pide(struct ioc * ioc,unsigned long * res_ptr,unsigned int bitshiftcnt)457b34eb53cSFUJITA Tomonori static unsigned long ptr_to_pide(struct ioc *ioc, unsigned long *res_ptr,
458b34eb53cSFUJITA Tomonori unsigned int bitshiftcnt)
459b34eb53cSFUJITA Tomonori {
460b34eb53cSFUJITA Tomonori return (((unsigned long)res_ptr - (unsigned long)ioc->res_map) << 3)
461b34eb53cSFUJITA Tomonori + bitshiftcnt;
462b34eb53cSFUJITA Tomonori }
463b34eb53cSFUJITA Tomonori
4641da177e4SLinus Torvalds /**
4651da177e4SLinus Torvalds * sba_search_bitmap - find free space in IO PDIR resource bitmap
4661da177e4SLinus Torvalds * @ioc: IO MMU structure which owns the pdir we are interested in.
4671da177e4SLinus Torvalds * @bits_wanted: number of entries we need.
4685f6602a1SAlex Williamson * @use_hint: use res_hint to indicate where to start looking
4691da177e4SLinus Torvalds *
4701da177e4SLinus Torvalds * Find consecutive free bits in resource bitmap.
4711da177e4SLinus Torvalds * Each bit represents one entry in the IO Pdir.
4721da177e4SLinus Torvalds * Cool perf optimization: search for log2(size) bits at a time.
4731da177e4SLinus Torvalds */
4741da177e4SLinus Torvalds static SBA_INLINE unsigned long
sba_search_bitmap(struct ioc * ioc,struct device * dev,unsigned long bits_wanted,int use_hint)475b34eb53cSFUJITA Tomonori sba_search_bitmap(struct ioc *ioc, struct device *dev,
476b34eb53cSFUJITA Tomonori unsigned long bits_wanted, int use_hint)
4771da177e4SLinus Torvalds {
4785f6602a1SAlex Williamson unsigned long *res_ptr;
4791da177e4SLinus Torvalds unsigned long *res_end = (unsigned long *) &(ioc->res_map[ioc->res_size]);
480b34eb53cSFUJITA Tomonori unsigned long flags, pide = ~0UL, tpide;
481b34eb53cSFUJITA Tomonori unsigned long boundary_size;
482b34eb53cSFUJITA Tomonori unsigned long shift;
483b34eb53cSFUJITA Tomonori int ret;
4841da177e4SLinus Torvalds
4851da177e4SLinus Torvalds ASSERT(((unsigned long) ioc->res_hint & (sizeof(unsigned long) - 1UL)) == 0);
4861da177e4SLinus Torvalds ASSERT(res_ptr < res_end);
4871da177e4SLinus Torvalds
4881e9d90dbSNicolin Chen boundary_size = dma_get_seg_boundary_nr_pages(dev, iovp_shift);
489b34eb53cSFUJITA Tomonori
490b34eb53cSFUJITA Tomonori BUG_ON(ioc->ibase & ~iovp_mask);
491b34eb53cSFUJITA Tomonori shift = ioc->ibase >> iovp_shift;
492b34eb53cSFUJITA Tomonori
4935f6602a1SAlex Williamson spin_lock_irqsave(&ioc->res_lock, flags);
4945f6602a1SAlex Williamson
4955f6602a1SAlex Williamson /* Allow caller to force a search through the entire resource space */
4965f6602a1SAlex Williamson if (likely(use_hint)) {
4975f6602a1SAlex Williamson res_ptr = ioc->res_hint;
4985f6602a1SAlex Williamson } else {
4995f6602a1SAlex Williamson res_ptr = (ulong *)ioc->res_map;
5005f6602a1SAlex Williamson ioc->res_bitshift = 0;
5015f6602a1SAlex Williamson }
5025f6602a1SAlex Williamson
5031da177e4SLinus Torvalds /*
5041da177e4SLinus Torvalds * N.B. REO/Grande defect AR2305 can cause TLB fetch timeouts
5051da177e4SLinus Torvalds * if a TLB entry is purged while in use. sba_mark_invalid()
5061da177e4SLinus Torvalds * purges IOTLB entries in power-of-two sizes, so we also
5071da177e4SLinus Torvalds * allocate IOVA space in power-of-two sizes.
5081da177e4SLinus Torvalds */
5091da177e4SLinus Torvalds bits_wanted = 1UL << get_iovp_order(bits_wanted << iovp_shift);
5101da177e4SLinus Torvalds
5111da177e4SLinus Torvalds if (likely(bits_wanted == 1)) {
5121da177e4SLinus Torvalds unsigned int bitshiftcnt;
5131da177e4SLinus Torvalds for(; res_ptr < res_end ; res_ptr++) {
5141da177e4SLinus Torvalds if (likely(*res_ptr != ~0UL)) {
5151da177e4SLinus Torvalds bitshiftcnt = ffz(*res_ptr);
5161da177e4SLinus Torvalds *res_ptr |= (1UL << bitshiftcnt);
517b34eb53cSFUJITA Tomonori pide = ptr_to_pide(ioc, res_ptr, bitshiftcnt);
5181da177e4SLinus Torvalds ioc->res_bitshift = bitshiftcnt + bits_wanted;
5191da177e4SLinus Torvalds goto found_it;
5201da177e4SLinus Torvalds }
5211da177e4SLinus Torvalds }
5221da177e4SLinus Torvalds goto not_found;
5231da177e4SLinus Torvalds
5241da177e4SLinus Torvalds }
5251da177e4SLinus Torvalds
5261da177e4SLinus Torvalds if (likely(bits_wanted <= BITS_PER_LONG/2)) {
5271da177e4SLinus Torvalds /*
5281da177e4SLinus Torvalds ** Search the resource bit map on well-aligned values.
5291da177e4SLinus Torvalds ** "o" is the alignment.
5301da177e4SLinus Torvalds ** We need the alignment to invalidate I/O TLB using
5311da177e4SLinus Torvalds ** SBA HW features in the unmap path.
5321da177e4SLinus Torvalds */
5331da177e4SLinus Torvalds unsigned long o = 1 << get_iovp_order(bits_wanted << iovp_shift);
5341da177e4SLinus Torvalds uint bitshiftcnt = ROUNDUP(ioc->res_bitshift, o);
5351da177e4SLinus Torvalds unsigned long mask, base_mask;
5361da177e4SLinus Torvalds
5371da177e4SLinus Torvalds base_mask = RESMAP_MASK(bits_wanted);
5381da177e4SLinus Torvalds mask = base_mask << bitshiftcnt;
5391da177e4SLinus Torvalds
540d4ed8084SHarvey Harrison DBG_RES("%s() o %ld %p", __func__, o, res_ptr);
5411da177e4SLinus Torvalds for(; res_ptr < res_end ; res_ptr++)
5421da177e4SLinus Torvalds {
5431da177e4SLinus Torvalds DBG_RES(" %p %lx %lx\n", res_ptr, mask, *res_ptr);
5441da177e4SLinus Torvalds ASSERT(0 != mask);
5451da177e4SLinus Torvalds for (; mask ; mask <<= o, bitshiftcnt += o) {
546b34eb53cSFUJITA Tomonori tpide = ptr_to_pide(ioc, res_ptr, bitshiftcnt);
547b34eb53cSFUJITA Tomonori ret = iommu_is_span_boundary(tpide, bits_wanted,
548b34eb53cSFUJITA Tomonori shift,
549b34eb53cSFUJITA Tomonori boundary_size);
550b34eb53cSFUJITA Tomonori if ((0 == ((*res_ptr) & mask)) && !ret) {
5511da177e4SLinus Torvalds *res_ptr |= mask; /* mark resources busy! */
552b34eb53cSFUJITA Tomonori pide = tpide;
5531da177e4SLinus Torvalds ioc->res_bitshift = bitshiftcnt + bits_wanted;
5541da177e4SLinus Torvalds goto found_it;
5551da177e4SLinus Torvalds }
5561da177e4SLinus Torvalds }
5571da177e4SLinus Torvalds
5581da177e4SLinus Torvalds bitshiftcnt = 0;
5591da177e4SLinus Torvalds mask = base_mask;
5601da177e4SLinus Torvalds
5611da177e4SLinus Torvalds }
5621da177e4SLinus Torvalds
5631da177e4SLinus Torvalds } else {
5641da177e4SLinus Torvalds int qwords, bits, i;
5651da177e4SLinus Torvalds unsigned long *end;
5661da177e4SLinus Torvalds
5671da177e4SLinus Torvalds qwords = bits_wanted >> 6; /* /64 */
5681da177e4SLinus Torvalds bits = bits_wanted - (qwords * BITS_PER_LONG);
5691da177e4SLinus Torvalds
5701da177e4SLinus Torvalds end = res_end - qwords;
5711da177e4SLinus Torvalds
5721da177e4SLinus Torvalds for (; res_ptr < end; res_ptr++) {
573b34eb53cSFUJITA Tomonori tpide = ptr_to_pide(ioc, res_ptr, 0);
574b34eb53cSFUJITA Tomonori ret = iommu_is_span_boundary(tpide, bits_wanted,
575b34eb53cSFUJITA Tomonori shift, boundary_size);
576b34eb53cSFUJITA Tomonori if (ret)
577b34eb53cSFUJITA Tomonori goto next_ptr;
5781da177e4SLinus Torvalds for (i = 0 ; i < qwords ; i++) {
5791da177e4SLinus Torvalds if (res_ptr[i] != 0)
5801da177e4SLinus Torvalds goto next_ptr;
5811da177e4SLinus Torvalds }
5821da177e4SLinus Torvalds if (bits && res_ptr[i] && (__ffs(res_ptr[i]) < bits))
5831da177e4SLinus Torvalds continue;
5841da177e4SLinus Torvalds
5851da177e4SLinus Torvalds /* Found it, mark it */
5861da177e4SLinus Torvalds for (i = 0 ; i < qwords ; i++)
5871da177e4SLinus Torvalds res_ptr[i] = ~0UL;
5881da177e4SLinus Torvalds res_ptr[i] |= RESMAP_MASK(bits);
5891da177e4SLinus Torvalds
590b34eb53cSFUJITA Tomonori pide = tpide;
5911da177e4SLinus Torvalds res_ptr += qwords;
5921da177e4SLinus Torvalds ioc->res_bitshift = bits;
5931da177e4SLinus Torvalds goto found_it;
5941da177e4SLinus Torvalds next_ptr:
5951da177e4SLinus Torvalds ;
5961da177e4SLinus Torvalds }
5971da177e4SLinus Torvalds }
5981da177e4SLinus Torvalds
5991da177e4SLinus Torvalds not_found:
6001da177e4SLinus Torvalds prefetch(ioc->res_map);
6011da177e4SLinus Torvalds ioc->res_hint = (unsigned long *) ioc->res_map;
6021da177e4SLinus Torvalds ioc->res_bitshift = 0;
6035f6602a1SAlex Williamson spin_unlock_irqrestore(&ioc->res_lock, flags);
6041da177e4SLinus Torvalds return (pide);
6051da177e4SLinus Torvalds
6061da177e4SLinus Torvalds found_it:
6071da177e4SLinus Torvalds ioc->res_hint = res_ptr;
6085f6602a1SAlex Williamson spin_unlock_irqrestore(&ioc->res_lock, flags);
6091da177e4SLinus Torvalds return (pide);
6101da177e4SLinus Torvalds }
6111da177e4SLinus Torvalds
6121da177e4SLinus Torvalds
6131da177e4SLinus Torvalds /**
6141da177e4SLinus Torvalds * sba_alloc_range - find free bits and mark them in IO PDIR resource bitmap
6151da177e4SLinus Torvalds * @ioc: IO MMU structure which owns the pdir we are interested in.
6161da177e4SLinus Torvalds * @size: number of bytes to create a mapping for
6171da177e4SLinus Torvalds *
6181da177e4SLinus Torvalds * Given a size, find consecutive unmarked and then mark those bits in the
6191da177e4SLinus Torvalds * resource bit map.
6201da177e4SLinus Torvalds */
6211da177e4SLinus Torvalds static int
sba_alloc_range(struct ioc * ioc,struct device * dev,size_t size)622b34eb53cSFUJITA Tomonori sba_alloc_range(struct ioc *ioc, struct device *dev, size_t size)
6231da177e4SLinus Torvalds {
6241da177e4SLinus Torvalds unsigned int pages_needed = size >> iovp_shift;
6251da177e4SLinus Torvalds #ifdef PDIR_SEARCH_TIMING
6261da177e4SLinus Torvalds unsigned long itc_start;
6271da177e4SLinus Torvalds #endif
6281da177e4SLinus Torvalds unsigned long pide;
6291da177e4SLinus Torvalds
6301da177e4SLinus Torvalds ASSERT(pages_needed);
6311da177e4SLinus Torvalds ASSERT(0 == (size & ~iovp_mask));
6321da177e4SLinus Torvalds
6331da177e4SLinus Torvalds #ifdef PDIR_SEARCH_TIMING
6341da177e4SLinus Torvalds itc_start = ia64_get_itc();
6351da177e4SLinus Torvalds #endif
6361da177e4SLinus Torvalds /*
6371da177e4SLinus Torvalds ** "seek and ye shall find"...praying never hurts either...
6381da177e4SLinus Torvalds */
639b34eb53cSFUJITA Tomonori pide = sba_search_bitmap(ioc, dev, pages_needed, 1);
6401da177e4SLinus Torvalds if (unlikely(pide >= (ioc->res_size << 3))) {
641b34eb53cSFUJITA Tomonori pide = sba_search_bitmap(ioc, dev, pages_needed, 0);
6421da177e4SLinus Torvalds if (unlikely(pide >= (ioc->res_size << 3))) {
6431da177e4SLinus Torvalds #if DELAYED_RESOURCE_CNT > 0
6445f6602a1SAlex Williamson unsigned long flags;
6455f6602a1SAlex Williamson
6461da177e4SLinus Torvalds /*
6471da177e4SLinus Torvalds ** With delayed resource freeing, we can give this one more shot. We're
6481da177e4SLinus Torvalds ** getting close to being in trouble here, so do what we can to make this
6491da177e4SLinus Torvalds ** one count.
6501da177e4SLinus Torvalds */
6515f6602a1SAlex Williamson spin_lock_irqsave(&ioc->saved_lock, flags);
6521da177e4SLinus Torvalds if (ioc->saved_cnt > 0) {
6531da177e4SLinus Torvalds struct sba_dma_pair *d;
6541da177e4SLinus Torvalds int cnt = ioc->saved_cnt;
6551da177e4SLinus Torvalds
6565f6602a1SAlex Williamson d = &(ioc->saved[ioc->saved_cnt - 1]);
6571da177e4SLinus Torvalds
6585f6602a1SAlex Williamson spin_lock(&ioc->res_lock);
6591da177e4SLinus Torvalds while (cnt--) {
6601da177e4SLinus Torvalds sba_mark_invalid(ioc, d->iova, d->size);
6611da177e4SLinus Torvalds sba_free_range(ioc, d->iova, d->size);
6621da177e4SLinus Torvalds d--;
6631da177e4SLinus Torvalds }
6641da177e4SLinus Torvalds ioc->saved_cnt = 0;
6651da177e4SLinus Torvalds READ_REG(ioc->ioc_hpa+IOC_PCOM); /* flush purges */
6665f6602a1SAlex Williamson spin_unlock(&ioc->res_lock);
6671da177e4SLinus Torvalds }
6685f6602a1SAlex Williamson spin_unlock_irqrestore(&ioc->saved_lock, flags);
6691da177e4SLinus Torvalds
670b34eb53cSFUJITA Tomonori pide = sba_search_bitmap(ioc, dev, pages_needed, 0);
671e2a46567SFUJITA Tomonori if (unlikely(pide >= (ioc->res_size << 3))) {
672e2a46567SFUJITA Tomonori printk(KERN_WARNING "%s: I/O MMU @ %p is"
673e2a46567SFUJITA Tomonori "out of mapping resources, %u %u %lx\n",
674e2a46567SFUJITA Tomonori __func__, ioc->ioc_hpa, ioc->res_size,
675e2a46567SFUJITA Tomonori pages_needed, dma_get_seg_boundary(dev));
676e2a46567SFUJITA Tomonori return -1;
677e2a46567SFUJITA Tomonori }
6781da177e4SLinus Torvalds #else
679e2a46567SFUJITA Tomonori printk(KERN_WARNING "%s: I/O MMU @ %p is"
680e2a46567SFUJITA Tomonori "out of mapping resources, %u %u %lx\n",
681e2a46567SFUJITA Tomonori __func__, ioc->ioc_hpa, ioc->res_size,
682e2a46567SFUJITA Tomonori pages_needed, dma_get_seg_boundary(dev));
683e2a46567SFUJITA Tomonori return -1;
6841da177e4SLinus Torvalds #endif
6851da177e4SLinus Torvalds }
6861da177e4SLinus Torvalds }
6871da177e4SLinus Torvalds
6881da177e4SLinus Torvalds #ifdef PDIR_SEARCH_TIMING
6891da177e4SLinus Torvalds ioc->avg_search[ioc->avg_idx++] = (ia64_get_itc() - itc_start) / pages_needed;
6901da177e4SLinus Torvalds ioc->avg_idx &= SBA_SEARCH_SAMPLE - 1;
6911da177e4SLinus Torvalds #endif
6921da177e4SLinus Torvalds
6931da177e4SLinus Torvalds prefetchw(&(ioc->pdir_base[pide]));
6941da177e4SLinus Torvalds
6951da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
6961da177e4SLinus Torvalds /* verify the first enable bit is clear */
6971da177e4SLinus Torvalds if(0x00 != ((u8 *) ioc->pdir_base)[pide*PDIR_ENTRY_SIZE + 7]) {
6981da177e4SLinus Torvalds sba_dump_pdir_entry(ioc, "sba_search_bitmap() botched it?", pide);
6991da177e4SLinus Torvalds }
7001da177e4SLinus Torvalds #endif
7011da177e4SLinus Torvalds
7021da177e4SLinus Torvalds DBG_RES("%s(%x) %d -> %lx hint %x/%x\n",
703d4ed8084SHarvey Harrison __func__, size, pages_needed, pide,
7041da177e4SLinus Torvalds (uint) ((unsigned long) ioc->res_hint - (unsigned long) ioc->res_map),
7051da177e4SLinus Torvalds ioc->res_bitshift );
7061da177e4SLinus Torvalds
7071da177e4SLinus Torvalds return (pide);
7081da177e4SLinus Torvalds }
7091da177e4SLinus Torvalds
7101da177e4SLinus Torvalds
7111da177e4SLinus Torvalds /**
7121da177e4SLinus Torvalds * sba_free_range - unmark bits in IO PDIR resource bitmap
7131da177e4SLinus Torvalds * @ioc: IO MMU structure which owns the pdir we are interested in.
7141da177e4SLinus Torvalds * @iova: IO virtual address which was previously allocated.
7151da177e4SLinus Torvalds * @size: number of bytes to create a mapping for
7161da177e4SLinus Torvalds *
7171da177e4SLinus Torvalds * clear bits in the ioc's resource map
7181da177e4SLinus Torvalds */
7191da177e4SLinus Torvalds static SBA_INLINE void
sba_free_range(struct ioc * ioc,dma_addr_t iova,size_t size)7201da177e4SLinus Torvalds sba_free_range(struct ioc *ioc, dma_addr_t iova, size_t size)
7211da177e4SLinus Torvalds {
7221da177e4SLinus Torvalds unsigned long iovp = SBA_IOVP(ioc, iova);
7231da177e4SLinus Torvalds unsigned int pide = PDIR_INDEX(iovp);
7241da177e4SLinus Torvalds unsigned int ridx = pide >> 3; /* convert bit to byte address */
7251da177e4SLinus Torvalds unsigned long *res_ptr = (unsigned long *) &((ioc)->res_map[ridx & ~RESMAP_IDX_MASK]);
7261da177e4SLinus Torvalds int bits_not_wanted = size >> iovp_shift;
7271da177e4SLinus Torvalds unsigned long m;
7281da177e4SLinus Torvalds
7291da177e4SLinus Torvalds /* Round up to power-of-two size: see AR2305 note above */
7301da177e4SLinus Torvalds bits_not_wanted = 1UL << get_iovp_order(bits_not_wanted << iovp_shift);
7311da177e4SLinus Torvalds for (; bits_not_wanted > 0 ; res_ptr++) {
7321da177e4SLinus Torvalds
7331da177e4SLinus Torvalds if (unlikely(bits_not_wanted > BITS_PER_LONG)) {
7341da177e4SLinus Torvalds
7351da177e4SLinus Torvalds /* these mappings start 64bit aligned */
7361da177e4SLinus Torvalds *res_ptr = 0UL;
7371da177e4SLinus Torvalds bits_not_wanted -= BITS_PER_LONG;
7381da177e4SLinus Torvalds pide += BITS_PER_LONG;
7391da177e4SLinus Torvalds
7401da177e4SLinus Torvalds } else {
7411da177e4SLinus Torvalds
7421da177e4SLinus Torvalds /* 3-bits "bit" address plus 2 (or 3) bits for "byte" == bit in word */
7431da177e4SLinus Torvalds m = RESMAP_MASK(bits_not_wanted) << (pide & (BITS_PER_LONG - 1));
7441da177e4SLinus Torvalds bits_not_wanted = 0;
7451da177e4SLinus Torvalds
746d4ed8084SHarvey Harrison DBG_RES("%s( ,%x,%x) %x/%lx %x %p %lx\n", __func__, (uint) iova, size,
7471da177e4SLinus Torvalds bits_not_wanted, m, pide, res_ptr, *res_ptr);
7481da177e4SLinus Torvalds
7491da177e4SLinus Torvalds ASSERT(m != 0);
7501da177e4SLinus Torvalds ASSERT(bits_not_wanted);
7511da177e4SLinus Torvalds ASSERT((*res_ptr & m) == m); /* verify same bits are set */
7521da177e4SLinus Torvalds *res_ptr &= ~m;
7531da177e4SLinus Torvalds }
7541da177e4SLinus Torvalds }
7551da177e4SLinus Torvalds }
7561da177e4SLinus Torvalds
7571da177e4SLinus Torvalds
7581da177e4SLinus Torvalds /**************************************************************
7591da177e4SLinus Torvalds *
7601da177e4SLinus Torvalds * "Dynamic DMA Mapping" support (aka "Coherent I/O")
7611da177e4SLinus Torvalds *
7621da177e4SLinus Torvalds ***************************************************************/
7631da177e4SLinus Torvalds
7641da177e4SLinus Torvalds /**
7651da177e4SLinus Torvalds * sba_io_pdir_entry - fill in one IO PDIR entry
7661da177e4SLinus Torvalds * @pdir_ptr: pointer to IO PDIR entry
7671da177e4SLinus Torvalds * @vba: Virtual CPU address of buffer to map
7681da177e4SLinus Torvalds *
7691da177e4SLinus Torvalds * SBA Mapping Routine
7701da177e4SLinus Torvalds *
7711da177e4SLinus Torvalds * Given a virtual address (vba, arg1) sba_io_pdir_entry()
7721da177e4SLinus Torvalds * loads the I/O PDIR entry pointed to by pdir_ptr (arg0).
7731da177e4SLinus Torvalds * Each IO Pdir entry consists of 8 bytes as shown below
7741da177e4SLinus Torvalds * (LSB == bit 0):
7751da177e4SLinus Torvalds *
7761da177e4SLinus Torvalds * 63 40 11 7 0
7771da177e4SLinus Torvalds * +-+---------------------+----------------------------------+----+--------+
7781da177e4SLinus Torvalds * |V| U | PPN[39:12] | U | FF |
7791da177e4SLinus Torvalds * +-+---------------------+----------------------------------+----+--------+
7801da177e4SLinus Torvalds *
7811da177e4SLinus Torvalds * V == Valid Bit
7821da177e4SLinus Torvalds * U == Unused
7831da177e4SLinus Torvalds * PPN == Physical Page Number
7841da177e4SLinus Torvalds *
7851da177e4SLinus Torvalds * The physical address fields are filled with the results of virt_to_phys()
7861da177e4SLinus Torvalds * on the vba.
7871da177e4SLinus Torvalds */
7881da177e4SLinus Torvalds
7891da177e4SLinus Torvalds #if 1
7901da177e4SLinus Torvalds #define sba_io_pdir_entry(pdir_ptr, vba) *pdir_ptr = ((vba & ~0xE000000000000FFFULL) \
7911da177e4SLinus Torvalds | 0x8000000000000000ULL)
7921da177e4SLinus Torvalds #else
7931da177e4SLinus Torvalds void SBA_INLINE
sba_io_pdir_entry(u64 * pdir_ptr,unsigned long vba)7941da177e4SLinus Torvalds sba_io_pdir_entry(u64 *pdir_ptr, unsigned long vba)
7951da177e4SLinus Torvalds {
7961da177e4SLinus Torvalds *pdir_ptr = ((vba & ~0xE000000000000FFFULL) | 0x80000000000000FFULL);
7971da177e4SLinus Torvalds }
7981da177e4SLinus Torvalds #endif
7991da177e4SLinus Torvalds
8001da177e4SLinus Torvalds #ifdef ENABLE_MARK_CLEAN
801*87639783SMatthew Wilcox (Oracle) /*
8021da177e4SLinus Torvalds * Since DMA is i-cache coherent, any (complete) pages that were written via
8031da177e4SLinus Torvalds * DMA can be marked as "clean" so that lazy_mmu_prot_update() doesn't have to
8041da177e4SLinus Torvalds * flush them when they get mapped into an executable vm-area.
8051da177e4SLinus Torvalds */
mark_clean(void * addr,size_t size)806*87639783SMatthew Wilcox (Oracle) static void mark_clean(void *addr, size_t size)
8071da177e4SLinus Torvalds {
808*87639783SMatthew Wilcox (Oracle) struct folio *folio = virt_to_folio(addr);
809*87639783SMatthew Wilcox (Oracle) ssize_t left = size;
810*87639783SMatthew Wilcox (Oracle) size_t offset = offset_in_folio(folio, addr);
8111da177e4SLinus Torvalds
812*87639783SMatthew Wilcox (Oracle) if (offset) {
813*87639783SMatthew Wilcox (Oracle) left -= folio_size(folio) - offset;
814*87639783SMatthew Wilcox (Oracle) if (left <= 0)
815*87639783SMatthew Wilcox (Oracle) return;
816*87639783SMatthew Wilcox (Oracle) folio = folio_next(folio);
817*87639783SMatthew Wilcox (Oracle) }
818*87639783SMatthew Wilcox (Oracle)
819*87639783SMatthew Wilcox (Oracle) while (left >= folio_size(folio)) {
820*87639783SMatthew Wilcox (Oracle) left -= folio_size(folio);
821*87639783SMatthew Wilcox (Oracle) set_bit(PG_arch_1, &folio->flags);
822*87639783SMatthew Wilcox (Oracle) if (!left)
823*87639783SMatthew Wilcox (Oracle) break;
824*87639783SMatthew Wilcox (Oracle) folio = folio_next(folio);
8251da177e4SLinus Torvalds }
8261da177e4SLinus Torvalds }
8271da177e4SLinus Torvalds #endif
8281da177e4SLinus Torvalds
8291da177e4SLinus Torvalds /**
8301da177e4SLinus Torvalds * sba_mark_invalid - invalidate one or more IO PDIR entries
8311da177e4SLinus Torvalds * @ioc: IO MMU structure which owns the pdir we are interested in.
8321da177e4SLinus Torvalds * @iova: IO Virtual Address mapped earlier
8331da177e4SLinus Torvalds * @byte_cnt: number of bytes this mapping covers.
8341da177e4SLinus Torvalds *
8351da177e4SLinus Torvalds * Marking the IO PDIR entry(ies) as Invalid and invalidate
8361da177e4SLinus Torvalds * corresponding IO TLB entry. The PCOM (Purge Command Register)
8371da177e4SLinus Torvalds * is to purge stale entries in the IO TLB when unmapping entries.
8381da177e4SLinus Torvalds *
8391da177e4SLinus Torvalds * The PCOM register supports purging of multiple pages, with a minium
8401da177e4SLinus Torvalds * of 1 page and a maximum of 2GB. Hardware requires the address be
8411da177e4SLinus Torvalds * aligned to the size of the range being purged. The size of the range
8421da177e4SLinus Torvalds * must be a power of 2. The "Cool perf optimization" in the
8431da177e4SLinus Torvalds * allocation routine helps keep that true.
8441da177e4SLinus Torvalds */
8451da177e4SLinus Torvalds static SBA_INLINE void
sba_mark_invalid(struct ioc * ioc,dma_addr_t iova,size_t byte_cnt)8461da177e4SLinus Torvalds sba_mark_invalid(struct ioc *ioc, dma_addr_t iova, size_t byte_cnt)
8471da177e4SLinus Torvalds {
8481da177e4SLinus Torvalds u32 iovp = (u32) SBA_IOVP(ioc,iova);
8491da177e4SLinus Torvalds
8501da177e4SLinus Torvalds int off = PDIR_INDEX(iovp);
8511da177e4SLinus Torvalds
8521da177e4SLinus Torvalds /* Must be non-zero and rounded up */
8531da177e4SLinus Torvalds ASSERT(byte_cnt > 0);
8541da177e4SLinus Torvalds ASSERT(0 == (byte_cnt & ~iovp_mask));
8551da177e4SLinus Torvalds
8561da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
8571da177e4SLinus Torvalds /* Assert first pdir entry is set */
8581da177e4SLinus Torvalds if (!(ioc->pdir_base[off] >> 60)) {
8591da177e4SLinus Torvalds sba_dump_pdir_entry(ioc,"sba_mark_invalid()", PDIR_INDEX(iovp));
8601da177e4SLinus Torvalds }
8611da177e4SLinus Torvalds #endif
8621da177e4SLinus Torvalds
8631da177e4SLinus Torvalds if (byte_cnt <= iovp_size)
8641da177e4SLinus Torvalds {
8651da177e4SLinus Torvalds ASSERT(off < ioc->pdir_size);
8661da177e4SLinus Torvalds
8671da177e4SLinus Torvalds iovp |= iovp_shift; /* set "size" field for PCOM */
8681da177e4SLinus Torvalds
8691da177e4SLinus Torvalds #ifndef FULL_VALID_PDIR
8701da177e4SLinus Torvalds /*
8711da177e4SLinus Torvalds ** clear I/O PDIR entry "valid" bit
8721da177e4SLinus Torvalds ** Do NOT clear the rest - save it for debugging.
8731da177e4SLinus Torvalds ** We should only clear bits that have previously
8741da177e4SLinus Torvalds ** been enabled.
8751da177e4SLinus Torvalds */
8761da177e4SLinus Torvalds ioc->pdir_base[off] &= ~(0x80000000000000FFULL);
8771da177e4SLinus Torvalds #else
8781da177e4SLinus Torvalds /*
8791da177e4SLinus Torvalds ** If we want to maintain the PDIR as valid, put in
8801da177e4SLinus Torvalds ** the spill page so devices prefetching won't
8811da177e4SLinus Torvalds ** cause a hard fail.
8821da177e4SLinus Torvalds */
8831da177e4SLinus Torvalds ioc->pdir_base[off] = (0x80000000000000FFULL | prefetch_spill_page);
8841da177e4SLinus Torvalds #endif
8851da177e4SLinus Torvalds } else {
8861da177e4SLinus Torvalds u32 t = get_iovp_order(byte_cnt) + iovp_shift;
8871da177e4SLinus Torvalds
8881da177e4SLinus Torvalds iovp |= t;
8891da177e4SLinus Torvalds ASSERT(t <= 31); /* 2GB! Max value of "size" field */
8901da177e4SLinus Torvalds
8911da177e4SLinus Torvalds do {
8921da177e4SLinus Torvalds /* verify this pdir entry is enabled */
8931da177e4SLinus Torvalds ASSERT(ioc->pdir_base[off] >> 63);
8941da177e4SLinus Torvalds #ifndef FULL_VALID_PDIR
8951da177e4SLinus Torvalds /* clear I/O Pdir entry "valid" bit first */
8961da177e4SLinus Torvalds ioc->pdir_base[off] &= ~(0x80000000000000FFULL);
8971da177e4SLinus Torvalds #else
8981da177e4SLinus Torvalds ioc->pdir_base[off] = (0x80000000000000FFULL | prefetch_spill_page);
8991da177e4SLinus Torvalds #endif
9001da177e4SLinus Torvalds off++;
9011da177e4SLinus Torvalds byte_cnt -= iovp_size;
9021da177e4SLinus Torvalds } while (byte_cnt > 0);
9031da177e4SLinus Torvalds }
9041da177e4SLinus Torvalds
9051da177e4SLinus Torvalds WRITE_REG(iovp | ioc->ibase, ioc->ioc_hpa+IOC_PCOM);
9061da177e4SLinus Torvalds }
9071da177e4SLinus Torvalds
9081da177e4SLinus Torvalds /**
90952aee3e8SChristoph Hellwig * sba_map_page - map one buffer and return IOVA for DMA
9101da177e4SLinus Torvalds * @dev: instance of PCI owned by the driver that's asking.
91152aee3e8SChristoph Hellwig * @page: page to map
91252aee3e8SChristoph Hellwig * @poff: offset into page
91352aee3e8SChristoph Hellwig * @size: number of bytes to map
91452aee3e8SChristoph Hellwig * @dir: dma direction
915309df0c5SArthur Kepner * @attrs: optional dma attributes
9161da177e4SLinus Torvalds *
917985098a0SMauro Carvalho Chehab * See Documentation/core-api/dma-api-howto.rst
9181da177e4SLinus Torvalds */
sba_map_page(struct device * dev,struct page * page,unsigned long poff,size_t size,enum dma_data_direction dir,unsigned long attrs)919160c1d8eSFUJITA Tomonori static dma_addr_t sba_map_page(struct device *dev, struct page *page,
920160c1d8eSFUJITA Tomonori unsigned long poff, size_t size,
921160c1d8eSFUJITA Tomonori enum dma_data_direction dir,
92200085f1eSKrzysztof Kozlowski unsigned long attrs)
9231da177e4SLinus Torvalds {
9241da177e4SLinus Torvalds struct ioc *ioc;
925160c1d8eSFUJITA Tomonori void *addr = page_address(page) + poff;
9261da177e4SLinus Torvalds dma_addr_t iovp;
9271da177e4SLinus Torvalds dma_addr_t offset;
9281da177e4SLinus Torvalds u64 *pdir_start;
9291da177e4SLinus Torvalds int pide;
9301da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
9311da177e4SLinus Torvalds unsigned long flags;
9321da177e4SLinus Torvalds #endif
9331da177e4SLinus Torvalds #ifdef ALLOW_IOV_BYPASS
9341da177e4SLinus Torvalds unsigned long pci_addr = virt_to_phys(addr);
9351da177e4SLinus Torvalds #endif
9361da177e4SLinus Torvalds
9371da177e4SLinus Torvalds #ifdef ALLOW_IOV_BYPASS
9381da177e4SLinus Torvalds ASSERT(to_pci_dev(dev)->dma_mask);
9391da177e4SLinus Torvalds /*
9401da177e4SLinus Torvalds ** Check if the PCI device can DMA to ptr... if so, just return ptr
9411da177e4SLinus Torvalds */
9421da177e4SLinus Torvalds if (likely((pci_addr & ~to_pci_dev(dev)->dma_mask) == 0)) {
9431da177e4SLinus Torvalds /*
9441da177e4SLinus Torvalds ** Device is bit capable of DMA'ing to the buffer...
9451da177e4SLinus Torvalds ** just return the PCI address of ptr
9461da177e4SLinus Torvalds */
94752aee3e8SChristoph Hellwig DBG_BYPASS("sba_map_page() bypass mask/addr: "
948309df0c5SArthur Kepner "0x%lx/0x%lx\n",
9491da177e4SLinus Torvalds to_pci_dev(dev)->dma_mask, pci_addr);
9501da177e4SLinus Torvalds return pci_addr;
9511da177e4SLinus Torvalds }
9521da177e4SLinus Torvalds #endif
9531da177e4SLinus Torvalds ioc = GET_IOC(dev);
9541da177e4SLinus Torvalds ASSERT(ioc);
9551da177e4SLinus Torvalds
9561da177e4SLinus Torvalds prefetch(ioc->res_hint);
9571da177e4SLinus Torvalds
9581da177e4SLinus Torvalds ASSERT(size > 0);
9591da177e4SLinus Torvalds ASSERT(size <= DMA_CHUNK_SIZE);
9601da177e4SLinus Torvalds
9611da177e4SLinus Torvalds /* save offset bits */
9621da177e4SLinus Torvalds offset = ((dma_addr_t) (long) addr) & ~iovp_mask;
9631da177e4SLinus Torvalds
9641da177e4SLinus Torvalds /* round up to nearest iovp_size */
9651da177e4SLinus Torvalds size = (size + offset + ~iovp_mask) & iovp_mask;
9661da177e4SLinus Torvalds
9671da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
9681da177e4SLinus Torvalds spin_lock_irqsave(&ioc->res_lock, flags);
96952aee3e8SChristoph Hellwig if (sba_check_pdir(ioc,"Check before sba_map_page()"))
9701da177e4SLinus Torvalds panic("Sanity check failed");
9711da177e4SLinus Torvalds spin_unlock_irqrestore(&ioc->res_lock, flags);
9721da177e4SLinus Torvalds #endif
9731da177e4SLinus Torvalds
974b34eb53cSFUJITA Tomonori pide = sba_alloc_range(ioc, dev, size);
975e2a46567SFUJITA Tomonori if (pide < 0)
97607256950SChristoph Hellwig return DMA_MAPPING_ERROR;
9771da177e4SLinus Torvalds
9781da177e4SLinus Torvalds iovp = (dma_addr_t) pide << iovp_shift;
9791da177e4SLinus Torvalds
980d4ed8084SHarvey Harrison DBG_RUN("%s() 0x%p -> 0x%lx\n", __func__, addr, (long) iovp | offset);
9811da177e4SLinus Torvalds
9821da177e4SLinus Torvalds pdir_start = &(ioc->pdir_base[pide]);
9831da177e4SLinus Torvalds
9841da177e4SLinus Torvalds while (size > 0) {
9851da177e4SLinus Torvalds ASSERT(((u8 *)pdir_start)[7] == 0); /* verify availability */
9861da177e4SLinus Torvalds sba_io_pdir_entry(pdir_start, (unsigned long) addr);
9871da177e4SLinus Torvalds
9881da177e4SLinus Torvalds DBG_RUN(" pdir 0x%p %lx\n", pdir_start, *pdir_start);
9891da177e4SLinus Torvalds
9901da177e4SLinus Torvalds addr += iovp_size;
9911da177e4SLinus Torvalds size -= iovp_size;
9921da177e4SLinus Torvalds pdir_start++;
9931da177e4SLinus Torvalds }
9941da177e4SLinus Torvalds /* force pdir update */
9951da177e4SLinus Torvalds wmb();
9961da177e4SLinus Torvalds
9971da177e4SLinus Torvalds /* form complete address */
9981da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
9991da177e4SLinus Torvalds spin_lock_irqsave(&ioc->res_lock, flags);
100052aee3e8SChristoph Hellwig sba_check_pdir(ioc,"Check after sba_map_page()");
10011da177e4SLinus Torvalds spin_unlock_irqrestore(&ioc->res_lock, flags);
10021da177e4SLinus Torvalds #endif
10031da177e4SLinus Torvalds return SBA_IOVA(ioc, iovp, offset);
10041da177e4SLinus Torvalds }
10051da177e4SLinus Torvalds
10065f6602a1SAlex Williamson #ifdef ENABLE_MARK_CLEAN
10075f6602a1SAlex Williamson static SBA_INLINE void
sba_mark_clean(struct ioc * ioc,dma_addr_t iova,size_t size)10085f6602a1SAlex Williamson sba_mark_clean(struct ioc *ioc, dma_addr_t iova, size_t size)
10095f6602a1SAlex Williamson {
10105f6602a1SAlex Williamson u32 iovp = (u32) SBA_IOVP(ioc,iova);
10115f6602a1SAlex Williamson int off = PDIR_INDEX(iovp);
10125f6602a1SAlex Williamson void *addr;
10135f6602a1SAlex Williamson
10145f6602a1SAlex Williamson if (size <= iovp_size) {
10155f6602a1SAlex Williamson addr = phys_to_virt(ioc->pdir_base[off] &
10165f6602a1SAlex Williamson ~0xE000000000000FFFULL);
10175f6602a1SAlex Williamson mark_clean(addr, size);
10185f6602a1SAlex Williamson } else {
10195f6602a1SAlex Williamson do {
10205f6602a1SAlex Williamson addr = phys_to_virt(ioc->pdir_base[off] &
10215f6602a1SAlex Williamson ~0xE000000000000FFFULL);
10225f6602a1SAlex Williamson mark_clean(addr, min(size, iovp_size));
10235f6602a1SAlex Williamson off++;
10245f6602a1SAlex Williamson size -= iovp_size;
10255f6602a1SAlex Williamson } while (size > 0);
10265f6602a1SAlex Williamson }
10275f6602a1SAlex Williamson }
10285f6602a1SAlex Williamson #endif
10295f6602a1SAlex Williamson
10301da177e4SLinus Torvalds /**
103152aee3e8SChristoph Hellwig * sba_unmap_page - unmap one IOVA and free resources
10321da177e4SLinus Torvalds * @dev: instance of PCI owned by the driver that's asking.
10331da177e4SLinus Torvalds * @iova: IOVA of driver buffer previously mapped.
10341da177e4SLinus Torvalds * @size: number of bytes mapped in driver buffer.
10351da177e4SLinus Torvalds * @dir: R/W or both.
1036309df0c5SArthur Kepner * @attrs: optional dma attributes
10371da177e4SLinus Torvalds *
1038985098a0SMauro Carvalho Chehab * See Documentation/core-api/dma-api-howto.rst
10391da177e4SLinus Torvalds */
sba_unmap_page(struct device * dev,dma_addr_t iova,size_t size,enum dma_data_direction dir,unsigned long attrs)1040160c1d8eSFUJITA Tomonori static void sba_unmap_page(struct device *dev, dma_addr_t iova, size_t size,
104100085f1eSKrzysztof Kozlowski enum dma_data_direction dir, unsigned long attrs)
10421da177e4SLinus Torvalds {
10431da177e4SLinus Torvalds struct ioc *ioc;
10441da177e4SLinus Torvalds #if DELAYED_RESOURCE_CNT > 0
10451da177e4SLinus Torvalds struct sba_dma_pair *d;
10461da177e4SLinus Torvalds #endif
10471da177e4SLinus Torvalds unsigned long flags;
10481da177e4SLinus Torvalds dma_addr_t offset;
10491da177e4SLinus Torvalds
10501da177e4SLinus Torvalds ioc = GET_IOC(dev);
10511da177e4SLinus Torvalds ASSERT(ioc);
10521da177e4SLinus Torvalds
10531da177e4SLinus Torvalds #ifdef ALLOW_IOV_BYPASS
10541da177e4SLinus Torvalds if (likely((iova & ioc->imask) != ioc->ibase)) {
10551da177e4SLinus Torvalds /*
10561da177e4SLinus Torvalds ** Address does not fall w/in IOVA, must be bypassing
10571da177e4SLinus Torvalds */
105852aee3e8SChristoph Hellwig DBG_BYPASS("sba_unmap_page() bypass addr: 0x%lx\n",
1059309df0c5SArthur Kepner iova);
10601da177e4SLinus Torvalds
10611da177e4SLinus Torvalds #ifdef ENABLE_MARK_CLEAN
10621da177e4SLinus Torvalds if (dir == DMA_FROM_DEVICE) {
10631da177e4SLinus Torvalds mark_clean(phys_to_virt(iova), size);
10641da177e4SLinus Torvalds }
10651da177e4SLinus Torvalds #endif
10661da177e4SLinus Torvalds return;
10671da177e4SLinus Torvalds }
10681da177e4SLinus Torvalds #endif
10691da177e4SLinus Torvalds offset = iova & ~iovp_mask;
10701da177e4SLinus Torvalds
1071d4ed8084SHarvey Harrison DBG_RUN("%s() iovp 0x%lx/%x\n", __func__, (long) iova, size);
10721da177e4SLinus Torvalds
10731da177e4SLinus Torvalds iova ^= offset; /* clear offset bits */
10741da177e4SLinus Torvalds size += offset;
10751da177e4SLinus Torvalds size = ROUNDUP(size, iovp_size);
10761da177e4SLinus Torvalds
10775f6602a1SAlex Williamson #ifdef ENABLE_MARK_CLEAN
10785f6602a1SAlex Williamson if (dir == DMA_FROM_DEVICE)
10795f6602a1SAlex Williamson sba_mark_clean(ioc, iova, size);
10805f6602a1SAlex Williamson #endif
10811da177e4SLinus Torvalds
10821da177e4SLinus Torvalds #if DELAYED_RESOURCE_CNT > 0
10831da177e4SLinus Torvalds spin_lock_irqsave(&ioc->saved_lock, flags);
10841da177e4SLinus Torvalds d = &(ioc->saved[ioc->saved_cnt]);
10851da177e4SLinus Torvalds d->iova = iova;
10861da177e4SLinus Torvalds d->size = size;
10871da177e4SLinus Torvalds if (unlikely(++(ioc->saved_cnt) >= DELAYED_RESOURCE_CNT)) {
10881da177e4SLinus Torvalds int cnt = ioc->saved_cnt;
10891da177e4SLinus Torvalds spin_lock(&ioc->res_lock);
10901da177e4SLinus Torvalds while (cnt--) {
10911da177e4SLinus Torvalds sba_mark_invalid(ioc, d->iova, d->size);
10921da177e4SLinus Torvalds sba_free_range(ioc, d->iova, d->size);
10931da177e4SLinus Torvalds d--;
10941da177e4SLinus Torvalds }
10951da177e4SLinus Torvalds ioc->saved_cnt = 0;
10961da177e4SLinus Torvalds READ_REG(ioc->ioc_hpa+IOC_PCOM); /* flush purges */
10971da177e4SLinus Torvalds spin_unlock(&ioc->res_lock);
10981da177e4SLinus Torvalds }
10991da177e4SLinus Torvalds spin_unlock_irqrestore(&ioc->saved_lock, flags);
11001da177e4SLinus Torvalds #else /* DELAYED_RESOURCE_CNT == 0 */
11011da177e4SLinus Torvalds spin_lock_irqsave(&ioc->res_lock, flags);
11021da177e4SLinus Torvalds sba_mark_invalid(ioc, iova, size);
11031da177e4SLinus Torvalds sba_free_range(ioc, iova, size);
11041da177e4SLinus Torvalds READ_REG(ioc->ioc_hpa+IOC_PCOM); /* flush purges */
11051da177e4SLinus Torvalds spin_unlock_irqrestore(&ioc->res_lock, flags);
11061da177e4SLinus Torvalds #endif /* DELAYED_RESOURCE_CNT == 0 */
11071da177e4SLinus Torvalds }
11081da177e4SLinus Torvalds
11091da177e4SLinus Torvalds /**
11101da177e4SLinus Torvalds * sba_alloc_coherent - allocate/map shared mem for DMA
11111da177e4SLinus Torvalds * @dev: instance of PCI owned by the driver that's asking.
11121da177e4SLinus Torvalds * @size: number of bytes mapped in driver buffer.
11131da177e4SLinus Torvalds * @dma_handle: IOVA of new buffer.
11141da177e4SLinus Torvalds *
1115985098a0SMauro Carvalho Chehab * See Documentation/core-api/dma-api-howto.rst
11161da177e4SLinus Torvalds */
1117055bcf99SFUJITA Tomonori static void *
sba_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t flags,unsigned long attrs)1118baa676fcSAndrzej Pietrasiewicz sba_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
111900085f1eSKrzysztof Kozlowski gfp_t flags, unsigned long attrs)
11201da177e4SLinus Torvalds {
112152aee3e8SChristoph Hellwig struct page *page;
11221da177e4SLinus Torvalds struct ioc *ioc;
112352aee3e8SChristoph Hellwig int node = -1;
11241da177e4SLinus Torvalds void *addr;
11251da177e4SLinus Torvalds
11261da177e4SLinus Torvalds ioc = GET_IOC(dev);
11271da177e4SLinus Torvalds ASSERT(ioc);
11281da177e4SLinus Torvalds #ifdef CONFIG_NUMA
112952aee3e8SChristoph Hellwig node = ioc->node;
113052aee3e8SChristoph Hellwig #endif
11311da177e4SLinus Torvalds
113252aee3e8SChristoph Hellwig page = alloc_pages_node(node, flags, get_order(size));
11331da177e4SLinus Torvalds if (unlikely(!page))
11341da177e4SLinus Torvalds return NULL;
11351da177e4SLinus Torvalds
11361da177e4SLinus Torvalds addr = page_address(page);
11371da177e4SLinus Torvalds memset(addr, 0, size);
113852aee3e8SChristoph Hellwig *dma_handle = page_to_phys(page);
11391da177e4SLinus Torvalds
11401da177e4SLinus Torvalds #ifdef ALLOW_IOV_BYPASS
11411da177e4SLinus Torvalds ASSERT(dev->coherent_dma_mask);
11421da177e4SLinus Torvalds /*
11431da177e4SLinus Torvalds ** Check if the PCI device can DMA to ptr... if so, just return ptr
11441da177e4SLinus Torvalds */
11451da177e4SLinus Torvalds if (likely((*dma_handle & ~dev->coherent_dma_mask) == 0)) {
11461da177e4SLinus Torvalds DBG_BYPASS("sba_alloc_coherent() bypass mask/addr: 0x%lx/0x%lx\n",
11471da177e4SLinus Torvalds dev->coherent_dma_mask, *dma_handle);
11481da177e4SLinus Torvalds
11491da177e4SLinus Torvalds return addr;
11501da177e4SLinus Torvalds }
11511da177e4SLinus Torvalds #endif
11521da177e4SLinus Torvalds
11531da177e4SLinus Torvalds /*
11541da177e4SLinus Torvalds * If device can't bypass or bypass is disabled, pass the 32bit fake
11551da177e4SLinus Torvalds * device to map single to get an iova mapping.
11561da177e4SLinus Torvalds */
115752aee3e8SChristoph Hellwig *dma_handle = sba_map_page(&ioc->sac_only_dev->dev, page, 0, size,
115852aee3e8SChristoph Hellwig DMA_BIDIRECTIONAL, 0);
115952aee3e8SChristoph Hellwig if (dma_mapping_error(dev, *dma_handle))
116052aee3e8SChristoph Hellwig return NULL;
11611da177e4SLinus Torvalds return addr;
11621da177e4SLinus Torvalds }
11631da177e4SLinus Torvalds
11641da177e4SLinus Torvalds
11651da177e4SLinus Torvalds /**
11661da177e4SLinus Torvalds * sba_free_coherent - free/unmap shared mem for DMA
11671da177e4SLinus Torvalds * @dev: instance of PCI owned by the driver that's asking.
11681da177e4SLinus Torvalds * @size: number of bytes mapped in driver buffer.
11691da177e4SLinus Torvalds * @vaddr: virtual address IOVA of "consistent" buffer.
11701da177e4SLinus Torvalds * @dma_handler: IO virtual address of "consistent" buffer.
11711da177e4SLinus Torvalds *
1172985098a0SMauro Carvalho Chehab * See Documentation/core-api/dma-api-howto.rst
11731da177e4SLinus Torvalds */
sba_free_coherent(struct device * dev,size_t size,void * vaddr,dma_addr_t dma_handle,unsigned long attrs)1174055bcf99SFUJITA Tomonori static void sba_free_coherent(struct device *dev, size_t size, void *vaddr,
117500085f1eSKrzysztof Kozlowski dma_addr_t dma_handle, unsigned long attrs)
11761da177e4SLinus Torvalds {
117752aee3e8SChristoph Hellwig sba_unmap_page(dev, dma_handle, size, 0, 0);
11781da177e4SLinus Torvalds free_pages((unsigned long) vaddr, get_order(size));
11791da177e4SLinus Torvalds }
11801da177e4SLinus Torvalds
11811da177e4SLinus Torvalds
11821da177e4SLinus Torvalds /*
11831da177e4SLinus Torvalds ** Since 0 is a valid pdir_base index value, can't use that
11841da177e4SLinus Torvalds ** to determine if a value is valid or not. Use a flag to indicate
11851da177e4SLinus Torvalds ** the SG list entry contains a valid pdir index.
11861da177e4SLinus Torvalds */
11871da177e4SLinus Torvalds #define PIDE_FLAG 0x1UL
11881da177e4SLinus Torvalds
11891da177e4SLinus Torvalds #ifdef DEBUG_LARGE_SG_ENTRIES
11901da177e4SLinus Torvalds int dump_run_sg = 0;
11911da177e4SLinus Torvalds #endif
11921da177e4SLinus Torvalds
11931da177e4SLinus Torvalds
11941da177e4SLinus Torvalds /**
11951da177e4SLinus Torvalds * sba_fill_pdir - write allocated SG entries into IO PDIR
11961da177e4SLinus Torvalds * @ioc: IO MMU structure which owns the pdir we are interested in.
11971da177e4SLinus Torvalds * @startsg: list of IOVA/size pairs
11981da177e4SLinus Torvalds * @nents: number of entries in startsg list
11991da177e4SLinus Torvalds *
12001da177e4SLinus Torvalds * Take preprocessed SG list and write corresponding entries
12011da177e4SLinus Torvalds * in the IO PDIR.
12021da177e4SLinus Torvalds */
12031da177e4SLinus Torvalds
12041da177e4SLinus Torvalds static SBA_INLINE int
sba_fill_pdir(struct ioc * ioc,struct scatterlist * startsg,int nents)12051da177e4SLinus Torvalds sba_fill_pdir(
12061da177e4SLinus Torvalds struct ioc *ioc,
12071da177e4SLinus Torvalds struct scatterlist *startsg,
12081da177e4SLinus Torvalds int nents)
12091da177e4SLinus Torvalds {
12101da177e4SLinus Torvalds struct scatterlist *dma_sg = startsg; /* pointer to current DMA */
12111da177e4SLinus Torvalds int n_mappings = 0;
12121da177e4SLinus Torvalds u64 *pdirp = NULL;
12131da177e4SLinus Torvalds unsigned long dma_offset = 0;
12141da177e4SLinus Torvalds
12151da177e4SLinus Torvalds while (nents-- > 0) {
12161da177e4SLinus Torvalds int cnt = startsg->dma_length;
12171da177e4SLinus Torvalds startsg->dma_length = 0;
12181da177e4SLinus Torvalds
12191da177e4SLinus Torvalds #ifdef DEBUG_LARGE_SG_ENTRIES
12201da177e4SLinus Torvalds if (dump_run_sg)
12211da177e4SLinus Torvalds printk(" %2d : %08lx/%05x %p\n",
12221da177e4SLinus Torvalds nents, startsg->dma_address, cnt,
12231da177e4SLinus Torvalds sba_sg_address(startsg));
12241da177e4SLinus Torvalds #else
12251da177e4SLinus Torvalds DBG_RUN_SG(" %d : %08lx/%05x %p\n",
12261da177e4SLinus Torvalds nents, startsg->dma_address, cnt,
12271da177e4SLinus Torvalds sba_sg_address(startsg));
12281da177e4SLinus Torvalds #endif
12291da177e4SLinus Torvalds /*
12301da177e4SLinus Torvalds ** Look for the start of a new DMA stream
12311da177e4SLinus Torvalds */
12321da177e4SLinus Torvalds if (startsg->dma_address & PIDE_FLAG) {
12331da177e4SLinus Torvalds u32 pide = startsg->dma_address & ~PIDE_FLAG;
12341da177e4SLinus Torvalds dma_offset = (unsigned long) pide & ~iovp_mask;
12351da177e4SLinus Torvalds startsg->dma_address = 0;
1236bdb02504SFUJITA Tomonori if (n_mappings)
12379b6eccfcSJens Axboe dma_sg = sg_next(dma_sg);
12381da177e4SLinus Torvalds dma_sg->dma_address = pide | ioc->ibase;
12391da177e4SLinus Torvalds pdirp = &(ioc->pdir_base[pide >> iovp_shift]);
12401da177e4SLinus Torvalds n_mappings++;
12411da177e4SLinus Torvalds }
12421da177e4SLinus Torvalds
12431da177e4SLinus Torvalds /*
12441da177e4SLinus Torvalds ** Look for a VCONTIG chunk
12451da177e4SLinus Torvalds */
12461da177e4SLinus Torvalds if (cnt) {
12471da177e4SLinus Torvalds unsigned long vaddr = (unsigned long) sba_sg_address(startsg);
12481da177e4SLinus Torvalds ASSERT(pdirp);
12491da177e4SLinus Torvalds
12501da177e4SLinus Torvalds /* Since multiple Vcontig blocks could make up
12511da177e4SLinus Torvalds ** one DMA stream, *add* cnt to dma_len.
12521da177e4SLinus Torvalds */
12531da177e4SLinus Torvalds dma_sg->dma_length += cnt;
12541da177e4SLinus Torvalds cnt += dma_offset;
12551da177e4SLinus Torvalds dma_offset=0; /* only want offset on first chunk */
12561da177e4SLinus Torvalds cnt = ROUNDUP(cnt, iovp_size);
12571da177e4SLinus Torvalds do {
12581da177e4SLinus Torvalds sba_io_pdir_entry(pdirp, vaddr);
12591da177e4SLinus Torvalds vaddr += iovp_size;
12601da177e4SLinus Torvalds cnt -= iovp_size;
12611da177e4SLinus Torvalds pdirp++;
12621da177e4SLinus Torvalds } while (cnt > 0);
12631da177e4SLinus Torvalds }
12649b6eccfcSJens Axboe startsg = sg_next(startsg);
12651da177e4SLinus Torvalds }
12661da177e4SLinus Torvalds /* force pdir update */
12671da177e4SLinus Torvalds wmb();
12681da177e4SLinus Torvalds
12691da177e4SLinus Torvalds #ifdef DEBUG_LARGE_SG_ENTRIES
12701da177e4SLinus Torvalds dump_run_sg = 0;
12711da177e4SLinus Torvalds #endif
12721da177e4SLinus Torvalds return(n_mappings);
12731da177e4SLinus Torvalds }
12741da177e4SLinus Torvalds
12751da177e4SLinus Torvalds
12761da177e4SLinus Torvalds /*
12771da177e4SLinus Torvalds ** Two address ranges are DMA contiguous *iff* "end of prev" and
12781da177e4SLinus Torvalds ** "start of next" are both on an IOV page boundary.
12791da177e4SLinus Torvalds **
12801da177e4SLinus Torvalds ** (shift left is a quick trick to mask off upper bits)
12811da177e4SLinus Torvalds */
12821da177e4SLinus Torvalds #define DMA_CONTIG(__X, __Y) \
12831da177e4SLinus Torvalds (((((unsigned long) __X) | ((unsigned long) __Y)) << (BITS_PER_LONG - iovp_shift)) == 0UL)
12841da177e4SLinus Torvalds
12851da177e4SLinus Torvalds
12861da177e4SLinus Torvalds /**
12871da177e4SLinus Torvalds * sba_coalesce_chunks - preprocess the SG list
12881da177e4SLinus Torvalds * @ioc: IO MMU structure which owns the pdir we are interested in.
12891da177e4SLinus Torvalds * @startsg: list of IOVA/size pairs
12901da177e4SLinus Torvalds * @nents: number of entries in startsg list
12911da177e4SLinus Torvalds *
12921da177e4SLinus Torvalds * First pass is to walk the SG list and determine where the breaks are
12931da177e4SLinus Torvalds * in the DMA stream. Allocates PDIR entries but does not fill them.
12941da177e4SLinus Torvalds * Returns the number of DMA chunks.
12951da177e4SLinus Torvalds *
12961da177e4SLinus Torvalds * Doing the fill separate from the coalescing/allocation keeps the
12971da177e4SLinus Torvalds * code simpler. Future enhancement could make one pass through
12981da177e4SLinus Torvalds * the sglist do both.
12991da177e4SLinus Torvalds */
13001da177e4SLinus Torvalds static SBA_INLINE int
sba_coalesce_chunks(struct ioc * ioc,struct device * dev,struct scatterlist * startsg,int nents)1301a031bbcbSFUJITA Tomonori sba_coalesce_chunks(struct ioc *ioc, struct device *dev,
13021da177e4SLinus Torvalds struct scatterlist *startsg,
13031da177e4SLinus Torvalds int nents)
13041da177e4SLinus Torvalds {
13051da177e4SLinus Torvalds struct scatterlist *vcontig_sg; /* VCONTIG chunk head */
13061da177e4SLinus Torvalds unsigned long vcontig_len; /* len of VCONTIG chunk */
13071da177e4SLinus Torvalds unsigned long vcontig_end;
13081da177e4SLinus Torvalds struct scatterlist *dma_sg; /* next DMA stream head */
13091da177e4SLinus Torvalds unsigned long dma_offset, dma_len; /* start/len of DMA stream */
13101da177e4SLinus Torvalds int n_mappings = 0;
1311a031bbcbSFUJITA Tomonori unsigned int max_seg_size = dma_get_max_seg_size(dev);
1312e2a46567SFUJITA Tomonori int idx;
13131da177e4SLinus Torvalds
13141da177e4SLinus Torvalds while (nents > 0) {
13151da177e4SLinus Torvalds unsigned long vaddr = (unsigned long) sba_sg_address(startsg);
13161da177e4SLinus Torvalds
13171da177e4SLinus Torvalds /*
13181da177e4SLinus Torvalds ** Prepare for first/next DMA stream
13191da177e4SLinus Torvalds */
13201da177e4SLinus Torvalds dma_sg = vcontig_sg = startsg;
13211da177e4SLinus Torvalds dma_len = vcontig_len = vcontig_end = startsg->length;
13221da177e4SLinus Torvalds vcontig_end += vaddr;
13231da177e4SLinus Torvalds dma_offset = vaddr & ~iovp_mask;
13241da177e4SLinus Torvalds
13251da177e4SLinus Torvalds /* PARANOID: clear entries */
13261da177e4SLinus Torvalds startsg->dma_address = startsg->dma_length = 0;
13271da177e4SLinus Torvalds
13281da177e4SLinus Torvalds /*
13291da177e4SLinus Torvalds ** This loop terminates one iteration "early" since
13301da177e4SLinus Torvalds ** it's always looking one "ahead".
13311da177e4SLinus Torvalds */
13321da177e4SLinus Torvalds while (--nents > 0) {
13331da177e4SLinus Torvalds unsigned long vaddr; /* tmp */
13341da177e4SLinus Torvalds
13359b6eccfcSJens Axboe startsg = sg_next(startsg);
13361da177e4SLinus Torvalds
13371da177e4SLinus Torvalds /* PARANOID */
13381da177e4SLinus Torvalds startsg->dma_address = startsg->dma_length = 0;
13391da177e4SLinus Torvalds
13401da177e4SLinus Torvalds /* catch brokenness in SCSI layer */
13411da177e4SLinus Torvalds ASSERT(startsg->length <= DMA_CHUNK_SIZE);
13421da177e4SLinus Torvalds
13431da177e4SLinus Torvalds /*
13441da177e4SLinus Torvalds ** First make sure current dma stream won't
13451da177e4SLinus Torvalds ** exceed DMA_CHUNK_SIZE if we coalesce the
13461da177e4SLinus Torvalds ** next entry.
13471da177e4SLinus Torvalds */
13481da177e4SLinus Torvalds if (((dma_len + dma_offset + startsg->length + ~iovp_mask) & iovp_mask)
13491da177e4SLinus Torvalds > DMA_CHUNK_SIZE)
13501da177e4SLinus Torvalds break;
13511da177e4SLinus Torvalds
1352a031bbcbSFUJITA Tomonori if (dma_len + startsg->length > max_seg_size)
1353a031bbcbSFUJITA Tomonori break;
1354a031bbcbSFUJITA Tomonori
13551da177e4SLinus Torvalds /*
13561da177e4SLinus Torvalds ** Then look for virtually contiguous blocks.
13571da177e4SLinus Torvalds **
13581da177e4SLinus Torvalds ** append the next transaction?
13591da177e4SLinus Torvalds */
13601da177e4SLinus Torvalds vaddr = (unsigned long) sba_sg_address(startsg);
13611da177e4SLinus Torvalds if (vcontig_end == vaddr)
13621da177e4SLinus Torvalds {
13631da177e4SLinus Torvalds vcontig_len += startsg->length;
13641da177e4SLinus Torvalds vcontig_end += startsg->length;
13651da177e4SLinus Torvalds dma_len += startsg->length;
13661da177e4SLinus Torvalds continue;
13671da177e4SLinus Torvalds }
13681da177e4SLinus Torvalds
13691da177e4SLinus Torvalds #ifdef DEBUG_LARGE_SG_ENTRIES
13701da177e4SLinus Torvalds dump_run_sg = (vcontig_len > iovp_size);
13711da177e4SLinus Torvalds #endif
13721da177e4SLinus Torvalds
13731da177e4SLinus Torvalds /*
1374af901ca1SAndré Goddard Rosa ** Not virtually contiguous.
13751da177e4SLinus Torvalds ** Terminate prev chunk.
13761da177e4SLinus Torvalds ** Start a new chunk.
13771da177e4SLinus Torvalds **
13781da177e4SLinus Torvalds ** Once we start a new VCONTIG chunk, dma_offset
13791da177e4SLinus Torvalds ** can't change. And we need the offset from the first
13801da177e4SLinus Torvalds ** chunk - not the last one. Ergo Successive chunks
13811da177e4SLinus Torvalds ** must start on page boundaries and dove tail
13821da177e4SLinus Torvalds ** with it's predecessor.
13831da177e4SLinus Torvalds */
13841da177e4SLinus Torvalds vcontig_sg->dma_length = vcontig_len;
13851da177e4SLinus Torvalds
13861da177e4SLinus Torvalds vcontig_sg = startsg;
13871da177e4SLinus Torvalds vcontig_len = startsg->length;
13881da177e4SLinus Torvalds
13891da177e4SLinus Torvalds /*
13901da177e4SLinus Torvalds ** 3) do the entries end/start on page boundaries?
13911da177e4SLinus Torvalds ** Don't update vcontig_end until we've checked.
13921da177e4SLinus Torvalds */
13931da177e4SLinus Torvalds if (DMA_CONTIG(vcontig_end, vaddr))
13941da177e4SLinus Torvalds {
13951da177e4SLinus Torvalds vcontig_end = vcontig_len + vaddr;
13961da177e4SLinus Torvalds dma_len += vcontig_len;
13971da177e4SLinus Torvalds continue;
13981da177e4SLinus Torvalds } else {
13991da177e4SLinus Torvalds break;
14001da177e4SLinus Torvalds }
14011da177e4SLinus Torvalds }
14021da177e4SLinus Torvalds
14031da177e4SLinus Torvalds /*
14041da177e4SLinus Torvalds ** End of DMA Stream
14051da177e4SLinus Torvalds ** Terminate last VCONTIG block.
14061da177e4SLinus Torvalds ** Allocate space for DMA stream.
14071da177e4SLinus Torvalds */
14081da177e4SLinus Torvalds vcontig_sg->dma_length = vcontig_len;
14091da177e4SLinus Torvalds dma_len = (dma_len + dma_offset + ~iovp_mask) & iovp_mask;
14101da177e4SLinus Torvalds ASSERT(dma_len <= DMA_CHUNK_SIZE);
1411e2a46567SFUJITA Tomonori idx = sba_alloc_range(ioc, dev, dma_len);
1412e2a46567SFUJITA Tomonori if (idx < 0) {
1413e2a46567SFUJITA Tomonori dma_sg->dma_length = 0;
1414e2a46567SFUJITA Tomonori return -1;
1415e2a46567SFUJITA Tomonori }
1416e2a46567SFUJITA Tomonori dma_sg->dma_address = (dma_addr_t)(PIDE_FLAG | (idx << iovp_shift)
14171da177e4SLinus Torvalds | dma_offset);
14181da177e4SLinus Torvalds n_mappings++;
14191da177e4SLinus Torvalds }
14201da177e4SLinus Torvalds
14211da177e4SLinus Torvalds return n_mappings;
14221da177e4SLinus Torvalds }
14231da177e4SLinus Torvalds
1424e2a46567SFUJITA Tomonori static void sba_unmap_sg_attrs(struct device *dev, struct scatterlist *sglist,
1425e2a46567SFUJITA Tomonori int nents, enum dma_data_direction dir,
142600085f1eSKrzysztof Kozlowski unsigned long attrs);
14271da177e4SLinus Torvalds /**
14281da177e4SLinus Torvalds * sba_map_sg - map Scatter/Gather list
14291da177e4SLinus Torvalds * @dev: instance of PCI owned by the driver that's asking.
14301da177e4SLinus Torvalds * @sglist: array of buffer/length pairs
14311da177e4SLinus Torvalds * @nents: number of entries in list
14321da177e4SLinus Torvalds * @dir: R/W or both.
1433309df0c5SArthur Kepner * @attrs: optional dma attributes
14341da177e4SLinus Torvalds *
1435985098a0SMauro Carvalho Chehab * See Documentation/core-api/dma-api-howto.rst
14361da177e4SLinus Torvalds */
sba_map_sg_attrs(struct device * dev,struct scatterlist * sglist,int nents,enum dma_data_direction dir,unsigned long attrs)1437055bcf99SFUJITA Tomonori static int sba_map_sg_attrs(struct device *dev, struct scatterlist *sglist,
1438160c1d8eSFUJITA Tomonori int nents, enum dma_data_direction dir,
143900085f1eSKrzysztof Kozlowski unsigned long attrs)
14401da177e4SLinus Torvalds {
14411da177e4SLinus Torvalds struct ioc *ioc;
14421da177e4SLinus Torvalds int coalesced, filled = 0;
14431da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
14441da177e4SLinus Torvalds unsigned long flags;
14451da177e4SLinus Torvalds #endif
14461da177e4SLinus Torvalds #ifdef ALLOW_IOV_BYPASS_SG
14471da177e4SLinus Torvalds struct scatterlist *sg;
14481da177e4SLinus Torvalds #endif
14491da177e4SLinus Torvalds
1450d4ed8084SHarvey Harrison DBG_RUN_SG("%s() START %d entries\n", __func__, nents);
14511da177e4SLinus Torvalds ioc = GET_IOC(dev);
14521da177e4SLinus Torvalds ASSERT(ioc);
14531da177e4SLinus Torvalds
14541da177e4SLinus Torvalds #ifdef ALLOW_IOV_BYPASS_SG
14551da177e4SLinus Torvalds ASSERT(to_pci_dev(dev)->dma_mask);
14561da177e4SLinus Torvalds if (likely((ioc->dma_mask & ~to_pci_dev(dev)->dma_mask) == 0)) {
14579b6eccfcSJens Axboe for_each_sg(sglist, sg, nents, filled) {
14581da177e4SLinus Torvalds sg->dma_length = sg->length;
14591da177e4SLinus Torvalds sg->dma_address = virt_to_phys(sba_sg_address(sg));
14601da177e4SLinus Torvalds }
14611da177e4SLinus Torvalds return filled;
14621da177e4SLinus Torvalds }
14631da177e4SLinus Torvalds #endif
14641da177e4SLinus Torvalds /* Fast path single entry scatterlists. */
14651da177e4SLinus Torvalds if (nents == 1) {
14661da177e4SLinus Torvalds sglist->dma_length = sglist->length;
146752aee3e8SChristoph Hellwig sglist->dma_address = sba_map_page(dev, sg_page(sglist),
146852aee3e8SChristoph Hellwig sglist->offset, sglist->length, dir, attrs);
146952aee3e8SChristoph Hellwig if (dma_mapping_error(dev, sglist->dma_address))
147062af5ca5SMartin Oliveira return -EIO;
14711da177e4SLinus Torvalds return 1;
14721da177e4SLinus Torvalds }
14731da177e4SLinus Torvalds
14741da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
14751da177e4SLinus Torvalds spin_lock_irqsave(&ioc->res_lock, flags);
1476309df0c5SArthur Kepner if (sba_check_pdir(ioc,"Check before sba_map_sg_attrs()"))
14771da177e4SLinus Torvalds {
14781da177e4SLinus Torvalds sba_dump_sg(ioc, sglist, nents);
1479309df0c5SArthur Kepner panic("Check before sba_map_sg_attrs()");
14801da177e4SLinus Torvalds }
14811da177e4SLinus Torvalds spin_unlock_irqrestore(&ioc->res_lock, flags);
14821da177e4SLinus Torvalds #endif
14831da177e4SLinus Torvalds
14841da177e4SLinus Torvalds prefetch(ioc->res_hint);
14851da177e4SLinus Torvalds
14861da177e4SLinus Torvalds /*
14871da177e4SLinus Torvalds ** First coalesce the chunks and allocate I/O pdir space
14881da177e4SLinus Torvalds **
14891da177e4SLinus Torvalds ** If this is one DMA stream, we can properly map using the
14901da177e4SLinus Torvalds ** correct virtual address associated with each DMA page.
14911da177e4SLinus Torvalds ** w/o this association, we wouldn't have coherent DMA!
14921da177e4SLinus Torvalds ** Access to the virtual address is what forces a two pass algorithm.
14931da177e4SLinus Torvalds */
1494a031bbcbSFUJITA Tomonori coalesced = sba_coalesce_chunks(ioc, dev, sglist, nents);
1495e2a46567SFUJITA Tomonori if (coalesced < 0) {
1496e2a46567SFUJITA Tomonori sba_unmap_sg_attrs(dev, sglist, nents, dir, attrs);
149762af5ca5SMartin Oliveira return -ENOMEM;
1498e2a46567SFUJITA Tomonori }
14991da177e4SLinus Torvalds
15001da177e4SLinus Torvalds /*
15011da177e4SLinus Torvalds ** Program the I/O Pdir
15021da177e4SLinus Torvalds **
15031da177e4SLinus Torvalds ** map the virtual addresses to the I/O Pdir
15041da177e4SLinus Torvalds ** o dma_address will contain the pdir index
15051da177e4SLinus Torvalds ** o dma_len will contain the number of bytes to map
15061da177e4SLinus Torvalds ** o address contains the virtual address.
15071da177e4SLinus Torvalds */
15081da177e4SLinus Torvalds filled = sba_fill_pdir(ioc, sglist, nents);
15091da177e4SLinus Torvalds
15101da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
15111da177e4SLinus Torvalds spin_lock_irqsave(&ioc->res_lock, flags);
1512309df0c5SArthur Kepner if (sba_check_pdir(ioc,"Check after sba_map_sg_attrs()"))
15131da177e4SLinus Torvalds {
15141da177e4SLinus Torvalds sba_dump_sg(ioc, sglist, nents);
1515309df0c5SArthur Kepner panic("Check after sba_map_sg_attrs()\n");
15161da177e4SLinus Torvalds }
15171da177e4SLinus Torvalds spin_unlock_irqrestore(&ioc->res_lock, flags);
15181da177e4SLinus Torvalds #endif
15191da177e4SLinus Torvalds
15201da177e4SLinus Torvalds ASSERT(coalesced == filled);
1521d4ed8084SHarvey Harrison DBG_RUN_SG("%s() DONE %d mappings\n", __func__, filled);
15221da177e4SLinus Torvalds
15231da177e4SLinus Torvalds return filled;
15241da177e4SLinus Torvalds }
15251da177e4SLinus Torvalds
15261da177e4SLinus Torvalds /**
1527309df0c5SArthur Kepner * sba_unmap_sg_attrs - unmap Scatter/Gather list
15281da177e4SLinus Torvalds * @dev: instance of PCI owned by the driver that's asking.
15291da177e4SLinus Torvalds * @sglist: array of buffer/length pairs
15301da177e4SLinus Torvalds * @nents: number of entries in list
15311da177e4SLinus Torvalds * @dir: R/W or both.
1532309df0c5SArthur Kepner * @attrs: optional dma attributes
15331da177e4SLinus Torvalds *
1534985098a0SMauro Carvalho Chehab * See Documentation/core-api/dma-api-howto.rst
15351da177e4SLinus Torvalds */
sba_unmap_sg_attrs(struct device * dev,struct scatterlist * sglist,int nents,enum dma_data_direction dir,unsigned long attrs)1536055bcf99SFUJITA Tomonori static void sba_unmap_sg_attrs(struct device *dev, struct scatterlist *sglist,
1537160c1d8eSFUJITA Tomonori int nents, enum dma_data_direction dir,
153800085f1eSKrzysztof Kozlowski unsigned long attrs)
15391da177e4SLinus Torvalds {
15401da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
15411da177e4SLinus Torvalds struct ioc *ioc;
15421da177e4SLinus Torvalds unsigned long flags;
15431da177e4SLinus Torvalds #endif
15441da177e4SLinus Torvalds
15451da177e4SLinus Torvalds DBG_RUN_SG("%s() START %d entries, %p,%x\n",
1546d4ed8084SHarvey Harrison __func__, nents, sba_sg_address(sglist), sglist->length);
15471da177e4SLinus Torvalds
15481da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
15491da177e4SLinus Torvalds ioc = GET_IOC(dev);
15501da177e4SLinus Torvalds ASSERT(ioc);
15511da177e4SLinus Torvalds
15521da177e4SLinus Torvalds spin_lock_irqsave(&ioc->res_lock, flags);
1553309df0c5SArthur Kepner sba_check_pdir(ioc,"Check before sba_unmap_sg_attrs()");
15541da177e4SLinus Torvalds spin_unlock_irqrestore(&ioc->res_lock, flags);
15551da177e4SLinus Torvalds #endif
15561da177e4SLinus Torvalds
15571da177e4SLinus Torvalds while (nents && sglist->dma_length) {
15581da177e4SLinus Torvalds
155952aee3e8SChristoph Hellwig sba_unmap_page(dev, sglist->dma_address, sglist->dma_length,
156052aee3e8SChristoph Hellwig dir, attrs);
15619b6eccfcSJens Axboe sglist = sg_next(sglist);
15621da177e4SLinus Torvalds nents--;
15631da177e4SLinus Torvalds }
15641da177e4SLinus Torvalds
1565d4ed8084SHarvey Harrison DBG_RUN_SG("%s() DONE (nents %d)\n", __func__, nents);
15661da177e4SLinus Torvalds
15671da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
15681da177e4SLinus Torvalds spin_lock_irqsave(&ioc->res_lock, flags);
1569309df0c5SArthur Kepner sba_check_pdir(ioc,"Check after sba_unmap_sg_attrs()");
15701da177e4SLinus Torvalds spin_unlock_irqrestore(&ioc->res_lock, flags);
15711da177e4SLinus Torvalds #endif
15721da177e4SLinus Torvalds
15731da177e4SLinus Torvalds }
15741da177e4SLinus Torvalds
15751da177e4SLinus Torvalds /**************************************************************
15761da177e4SLinus Torvalds *
15771da177e4SLinus Torvalds * Initialization and claim
15781da177e4SLinus Torvalds *
15791da177e4SLinus Torvalds ***************************************************************/
15801da177e4SLinus Torvalds
1581f3ffaaa8SJiang Liu static void
ioc_iova_init(struct ioc * ioc)15821da177e4SLinus Torvalds ioc_iova_init(struct ioc *ioc)
15831da177e4SLinus Torvalds {
15841da177e4SLinus Torvalds int tcnfg;
15851da177e4SLinus Torvalds int agp_found = 0;
15861da177e4SLinus Torvalds struct pci_dev *device = NULL;
15871da177e4SLinus Torvalds #ifdef FULL_VALID_PDIR
15881da177e4SLinus Torvalds unsigned long index;
15891da177e4SLinus Torvalds #endif
15901da177e4SLinus Torvalds
15911da177e4SLinus Torvalds /*
15921da177e4SLinus Torvalds ** Firmware programs the base and size of a "safe IOVA space"
15931da177e4SLinus Torvalds ** (one that doesn't overlap memory or LMMIO space) in the
15941da177e4SLinus Torvalds ** IBASE and IMASK registers.
15951da177e4SLinus Torvalds */
15961da177e4SLinus Torvalds ioc->ibase = READ_REG(ioc->ioc_hpa + IOC_IBASE) & ~0x1UL;
15971da177e4SLinus Torvalds ioc->imask = READ_REG(ioc->ioc_hpa + IOC_IMASK) | 0xFFFFFFFF00000000UL;
15981da177e4SLinus Torvalds
15991da177e4SLinus Torvalds ioc->iov_size = ~ioc->imask + 1;
16001da177e4SLinus Torvalds
16011da177e4SLinus Torvalds DBG_INIT("%s() hpa %p IOV base 0x%lx mask 0x%lx (%dMB)\n",
1602d4ed8084SHarvey Harrison __func__, ioc->ioc_hpa, ioc->ibase, ioc->imask,
16031da177e4SLinus Torvalds ioc->iov_size >> 20);
16041da177e4SLinus Torvalds
16051da177e4SLinus Torvalds switch (iovp_size) {
16061da177e4SLinus Torvalds case 4*1024: tcnfg = 0; break;
16071da177e4SLinus Torvalds case 8*1024: tcnfg = 1; break;
16081da177e4SLinus Torvalds case 16*1024: tcnfg = 2; break;
16091da177e4SLinus Torvalds case 64*1024: tcnfg = 3; break;
16101da177e4SLinus Torvalds default:
16111da177e4SLinus Torvalds panic(PFX "Unsupported IOTLB page size %ldK",
16121da177e4SLinus Torvalds iovp_size >> 10);
16131da177e4SLinus Torvalds break;
16141da177e4SLinus Torvalds }
16151da177e4SLinus Torvalds WRITE_REG(tcnfg, ioc->ioc_hpa + IOC_TCNFG);
16161da177e4SLinus Torvalds
16171da177e4SLinus Torvalds ioc->pdir_size = (ioc->iov_size / iovp_size) * PDIR_ENTRY_SIZE;
16181da177e4SLinus Torvalds ioc->pdir_base = (void *) __get_free_pages(GFP_KERNEL,
16191da177e4SLinus Torvalds get_order(ioc->pdir_size));
16201da177e4SLinus Torvalds if (!ioc->pdir_base)
16211da177e4SLinus Torvalds panic(PFX "Couldn't allocate I/O Page Table\n");
16221da177e4SLinus Torvalds
16231da177e4SLinus Torvalds memset(ioc->pdir_base, 0, ioc->pdir_size);
16241da177e4SLinus Torvalds
1625d4ed8084SHarvey Harrison DBG_INIT("%s() IOV page size %ldK pdir %p size %x\n", __func__,
16261da177e4SLinus Torvalds iovp_size >> 10, ioc->pdir_base, ioc->pdir_size);
16271da177e4SLinus Torvalds
16281da177e4SLinus Torvalds ASSERT(ALIGN((unsigned long) ioc->pdir_base, 4*1024) == (unsigned long) ioc->pdir_base);
16291da177e4SLinus Torvalds WRITE_REG(virt_to_phys(ioc->pdir_base), ioc->ioc_hpa + IOC_PDIR_BASE);
16301da177e4SLinus Torvalds
16311da177e4SLinus Torvalds /*
16321da177e4SLinus Torvalds ** If an AGP device is present, only use half of the IOV space
16331da177e4SLinus Torvalds ** for PCI DMA. Unfortunately we can't know ahead of time
16341da177e4SLinus Torvalds ** whether GART support will actually be used, for now we
16351da177e4SLinus Torvalds ** can just key on an AGP device found in the system.
16361da177e4SLinus Torvalds ** We program the next pdir index after we stop w/ a key for
16371da177e4SLinus Torvalds ** the GART code to handshake on.
16381da177e4SLinus Torvalds */
16391da177e4SLinus Torvalds for_each_pci_dev(device)
16401da177e4SLinus Torvalds agp_found |= pci_find_capability(device, PCI_CAP_ID_AGP);
16411da177e4SLinus Torvalds
16421da177e4SLinus Torvalds if (agp_found && reserve_sba_gart) {
16431da177e4SLinus Torvalds printk(KERN_INFO PFX "reserving %dMb of IOVA space at 0x%lx for agpgart\n",
16441da177e4SLinus Torvalds ioc->iov_size/2 >> 20, ioc->ibase + ioc->iov_size/2);
16451da177e4SLinus Torvalds ioc->pdir_size /= 2;
16461da177e4SLinus Torvalds ((u64 *)ioc->pdir_base)[PDIR_INDEX(ioc->iov_size/2)] = ZX1_SBA_IOMMU_COOKIE;
16471da177e4SLinus Torvalds }
16481da177e4SLinus Torvalds #ifdef FULL_VALID_PDIR
16491da177e4SLinus Torvalds /*
16501da177e4SLinus Torvalds ** Check to see if the spill page has been allocated, we don't need more than
16511da177e4SLinus Torvalds ** one across multiple SBAs.
16521da177e4SLinus Torvalds */
16531da177e4SLinus Torvalds if (!prefetch_spill_page) {
16541da177e4SLinus Torvalds char *spill_poison = "SBAIOMMU POISON";
16551da177e4SLinus Torvalds int poison_size = 16;
16561da177e4SLinus Torvalds void *poison_addr, *addr;
16571da177e4SLinus Torvalds
16581da177e4SLinus Torvalds addr = (void *)__get_free_pages(GFP_KERNEL, get_order(iovp_size));
16591da177e4SLinus Torvalds if (!addr)
16601da177e4SLinus Torvalds panic(PFX "Couldn't allocate PDIR spill page\n");
16611da177e4SLinus Torvalds
16621da177e4SLinus Torvalds poison_addr = addr;
16631da177e4SLinus Torvalds for ( ; (u64) poison_addr < addr + iovp_size; poison_addr += poison_size)
16641da177e4SLinus Torvalds memcpy(poison_addr, spill_poison, poison_size);
16651da177e4SLinus Torvalds
16661da177e4SLinus Torvalds prefetch_spill_page = virt_to_phys(addr);
16671da177e4SLinus Torvalds
1668d4ed8084SHarvey Harrison DBG_INIT("%s() prefetch spill addr: 0x%lx\n", __func__, prefetch_spill_page);
16691da177e4SLinus Torvalds }
16701da177e4SLinus Torvalds /*
16711da177e4SLinus Torvalds ** Set all the PDIR entries valid w/ the spill page as the target
16721da177e4SLinus Torvalds */
16731da177e4SLinus Torvalds for (index = 0 ; index < (ioc->pdir_size / PDIR_ENTRY_SIZE) ; index++)
16741da177e4SLinus Torvalds ((u64 *)ioc->pdir_base)[index] = (0x80000000000000FF | prefetch_spill_page);
16751da177e4SLinus Torvalds #endif
16761da177e4SLinus Torvalds
16771da177e4SLinus Torvalds /* Clear I/O TLB of any possible entries */
16781da177e4SLinus Torvalds WRITE_REG(ioc->ibase | (get_iovp_order(ioc->iov_size) + iovp_shift), ioc->ioc_hpa + IOC_PCOM);
16791da177e4SLinus Torvalds READ_REG(ioc->ioc_hpa + IOC_PCOM);
16801da177e4SLinus Torvalds
16811da177e4SLinus Torvalds /* Enable IOVA translation */
16821da177e4SLinus Torvalds WRITE_REG(ioc->ibase | 1, ioc->ioc_hpa + IOC_IBASE);
16831da177e4SLinus Torvalds READ_REG(ioc->ioc_hpa + IOC_IBASE);
16841da177e4SLinus Torvalds }
16851da177e4SLinus Torvalds
16861da177e4SLinus Torvalds static void __init
ioc_resource_init(struct ioc * ioc)16871da177e4SLinus Torvalds ioc_resource_init(struct ioc *ioc)
16881da177e4SLinus Torvalds {
16891da177e4SLinus Torvalds spin_lock_init(&ioc->res_lock);
16901da177e4SLinus Torvalds #if DELAYED_RESOURCE_CNT > 0
16911da177e4SLinus Torvalds spin_lock_init(&ioc->saved_lock);
16921da177e4SLinus Torvalds #endif
16931da177e4SLinus Torvalds
16941da177e4SLinus Torvalds /* resource map size dictated by pdir_size */
16951da177e4SLinus Torvalds ioc->res_size = ioc->pdir_size / PDIR_ENTRY_SIZE; /* entries */
16961da177e4SLinus Torvalds ioc->res_size >>= 3; /* convert bit count to byte count */
1697d4ed8084SHarvey Harrison DBG_INIT("%s() res_size 0x%x\n", __func__, ioc->res_size);
16981da177e4SLinus Torvalds
16991da177e4SLinus Torvalds ioc->res_map = (char *) __get_free_pages(GFP_KERNEL,
17001da177e4SLinus Torvalds get_order(ioc->res_size));
17011da177e4SLinus Torvalds if (!ioc->res_map)
17021da177e4SLinus Torvalds panic(PFX "Couldn't allocate resource map\n");
17031da177e4SLinus Torvalds
17041da177e4SLinus Torvalds memset(ioc->res_map, 0, ioc->res_size);
17051da177e4SLinus Torvalds /* next available IOVP - circular search */
17061da177e4SLinus Torvalds ioc->res_hint = (unsigned long *) ioc->res_map;
17071da177e4SLinus Torvalds
17081da177e4SLinus Torvalds #ifdef ASSERT_PDIR_SANITY
17091da177e4SLinus Torvalds /* Mark first bit busy - ie no IOVA 0 */
17101da177e4SLinus Torvalds ioc->res_map[0] = 0x1;
17111da177e4SLinus Torvalds ioc->pdir_base[0] = 0x8000000000000000ULL | ZX1_SBA_IOMMU_COOKIE;
17121da177e4SLinus Torvalds #endif
17131da177e4SLinus Torvalds #ifdef FULL_VALID_PDIR
17141da177e4SLinus Torvalds /* Mark the last resource used so we don't prefetch beyond IOVA space */
17151da177e4SLinus Torvalds ioc->res_map[ioc->res_size - 1] |= 0x80UL; /* res_map is chars */
17161da177e4SLinus Torvalds ioc->pdir_base[(ioc->pdir_size / PDIR_ENTRY_SIZE) - 1] = (0x80000000000000FF
17171da177e4SLinus Torvalds | prefetch_spill_page);
17181da177e4SLinus Torvalds #endif
17191da177e4SLinus Torvalds
1720d4ed8084SHarvey Harrison DBG_INIT("%s() res_map %x %p\n", __func__,
17211da177e4SLinus Torvalds ioc->res_size, (void *) ioc->res_map);
17221da177e4SLinus Torvalds }
17231da177e4SLinus Torvalds
17241da177e4SLinus Torvalds static void __init
ioc_sac_init(struct ioc * ioc)17251da177e4SLinus Torvalds ioc_sac_init(struct ioc *ioc)
17261da177e4SLinus Torvalds {
17271da177e4SLinus Torvalds struct pci_dev *sac = NULL;
17281da177e4SLinus Torvalds struct pci_controller *controller = NULL;
17291da177e4SLinus Torvalds
17301da177e4SLinus Torvalds /*
17311da177e4SLinus Torvalds * pci_alloc_coherent() must return a DMA address which is
17321da177e4SLinus Torvalds * SAC (single address cycle) addressable, so allocate a
17331da177e4SLinus Torvalds * pseudo-device to enforce that.
17341da177e4SLinus Torvalds */
173552fd9108SYan Burman sac = kzalloc(sizeof(*sac), GFP_KERNEL);
17361da177e4SLinus Torvalds if (!sac)
17371da177e4SLinus Torvalds panic(PFX "Couldn't allocate struct pci_dev");
17381da177e4SLinus Torvalds
173952fd9108SYan Burman controller = kzalloc(sizeof(*controller), GFP_KERNEL);
17401da177e4SLinus Torvalds if (!controller)
17411da177e4SLinus Torvalds panic(PFX "Couldn't allocate struct pci_controller");
17421da177e4SLinus Torvalds
17431da177e4SLinus Torvalds controller->iommu = ioc;
17441da177e4SLinus Torvalds sac->sysdata = controller;
17451da177e4SLinus Torvalds sac->dma_mask = 0xFFFFFFFFUL;
17461da177e4SLinus Torvalds sac->dev.bus = &pci_bus_type;
17471da177e4SLinus Torvalds ioc->sac_only_dev = sac;
17481da177e4SLinus Torvalds }
17491da177e4SLinus Torvalds
17501da177e4SLinus Torvalds static void __init
ioc_zx1_init(struct ioc * ioc)17511da177e4SLinus Torvalds ioc_zx1_init(struct ioc *ioc)
17521da177e4SLinus Torvalds {
17531da177e4SLinus Torvalds unsigned long rope_config;
17541da177e4SLinus Torvalds unsigned int i;
17551da177e4SLinus Torvalds
17561da177e4SLinus Torvalds if (ioc->rev < 0x20)
17571da177e4SLinus Torvalds panic(PFX "IOC 2.0 or later required for IOMMU support\n");
17581da177e4SLinus Torvalds
17591da177e4SLinus Torvalds /* 38 bit memory controller + extra bit for range displaced by MMIO */
17601da177e4SLinus Torvalds ioc->dma_mask = (0x1UL << 39) - 1;
17611da177e4SLinus Torvalds
17621da177e4SLinus Torvalds /*
17631da177e4SLinus Torvalds ** Clear ROPE(N)_CONFIG AO bit.
17641da177e4SLinus Torvalds ** Disables "NT Ordering" (~= !"Relaxed Ordering")
17651da177e4SLinus Torvalds ** Overrides bit 1 in DMA Hint Sets.
17661da177e4SLinus Torvalds ** Improves netperf UDP_STREAM by ~10% for tg3 on bcm5701.
17671da177e4SLinus Torvalds */
17681da177e4SLinus Torvalds for (i=0; i<(8*8); i+=8) {
17691da177e4SLinus Torvalds rope_config = READ_REG(ioc->ioc_hpa + IOC_ROPE0_CFG + i);
17701da177e4SLinus Torvalds rope_config &= ~IOC_ROPE_AO;
17711da177e4SLinus Torvalds WRITE_REG(rope_config, ioc->ioc_hpa + IOC_ROPE0_CFG + i);
17721da177e4SLinus Torvalds }
17731da177e4SLinus Torvalds }
17741da177e4SLinus Torvalds
17751da177e4SLinus Torvalds typedef void (initfunc)(struct ioc *);
17761da177e4SLinus Torvalds
17771da177e4SLinus Torvalds struct ioc_iommu {
17781da177e4SLinus Torvalds u32 func_id;
17791da177e4SLinus Torvalds char *name;
17801da177e4SLinus Torvalds initfunc *init;
17811da177e4SLinus Torvalds };
17821da177e4SLinus Torvalds
17831da177e4SLinus Torvalds static struct ioc_iommu ioc_iommu_info[] __initdata = {
17841da177e4SLinus Torvalds { ZX1_IOC_ID, "zx1", ioc_zx1_init },
17851da177e4SLinus Torvalds { ZX2_IOC_ID, "zx2", NULL },
17861da177e4SLinus Torvalds { SX1000_IOC_ID, "sx1000", NULL },
1787e15da401SBjorn Helgaas { SX2000_IOC_ID, "sx2000", NULL },
17881da177e4SLinus Torvalds };
17891da177e4SLinus Torvalds
ioc_init(unsigned long hpa,struct ioc * ioc)17902edd73a4STony Luck static void __init ioc_init(unsigned long hpa, struct ioc *ioc)
17911da177e4SLinus Torvalds {
17921da177e4SLinus Torvalds struct ioc_iommu *info;
17931da177e4SLinus Torvalds
17941da177e4SLinus Torvalds ioc->next = ioc_list;
17951da177e4SLinus Torvalds ioc_list = ioc;
17961da177e4SLinus Torvalds
17971da177e4SLinus Torvalds ioc->ioc_hpa = ioremap(hpa, 0x1000);
17981da177e4SLinus Torvalds
17991da177e4SLinus Torvalds ioc->func_id = READ_REG(ioc->ioc_hpa + IOC_FUNC_ID);
18001da177e4SLinus Torvalds ioc->rev = READ_REG(ioc->ioc_hpa + IOC_FCLASS) & 0xFFUL;
18011da177e4SLinus Torvalds ioc->dma_mask = 0xFFFFFFFFFFFFFFFFUL; /* conservative */
18021da177e4SLinus Torvalds
18031da177e4SLinus Torvalds for (info = ioc_iommu_info; info < ioc_iommu_info + ARRAY_SIZE(ioc_iommu_info); info++) {
18041da177e4SLinus Torvalds if (ioc->func_id == info->func_id) {
18051da177e4SLinus Torvalds ioc->name = info->name;
18061da177e4SLinus Torvalds if (info->init)
18071da177e4SLinus Torvalds (info->init)(ioc);
18081da177e4SLinus Torvalds }
18091da177e4SLinus Torvalds }
18101da177e4SLinus Torvalds
18111da177e4SLinus Torvalds iovp_size = (1 << iovp_shift);
18121da177e4SLinus Torvalds iovp_mask = ~(iovp_size - 1);
18131da177e4SLinus Torvalds
1814d4ed8084SHarvey Harrison DBG_INIT("%s: PAGE_SIZE %ldK, iovp_size %ldK\n", __func__,
18151da177e4SLinus Torvalds PAGE_SIZE >> 10, iovp_size >> 10);
18161da177e4SLinus Torvalds
18171da177e4SLinus Torvalds if (!ioc->name) {
18181da177e4SLinus Torvalds ioc->name = kmalloc(24, GFP_KERNEL);
18191da177e4SLinus Torvalds if (ioc->name)
18201da177e4SLinus Torvalds sprintf((char *) ioc->name, "Unknown (%04x:%04x)",
18211da177e4SLinus Torvalds ioc->func_id & 0xFFFF, (ioc->func_id >> 16) & 0xFFFF);
18221da177e4SLinus Torvalds else
18231da177e4SLinus Torvalds ioc->name = "Unknown";
18241da177e4SLinus Torvalds }
18251da177e4SLinus Torvalds
18261da177e4SLinus Torvalds ioc_iova_init(ioc);
18271da177e4SLinus Torvalds ioc_resource_init(ioc);
18281da177e4SLinus Torvalds ioc_sac_init(ioc);
18291da177e4SLinus Torvalds
18301da177e4SLinus Torvalds printk(KERN_INFO PFX
18311da177e4SLinus Torvalds "%s %d.%d HPA 0x%lx IOVA space %dMb at 0x%lx\n",
18321da177e4SLinus Torvalds ioc->name, (ioc->rev >> 4) & 0xF, ioc->rev & 0xF,
18331da177e4SLinus Torvalds hpa, ioc->iov_size >> 20, ioc->ibase);
18341da177e4SLinus Torvalds }
18351da177e4SLinus Torvalds
18361da177e4SLinus Torvalds
18371da177e4SLinus Torvalds
18381da177e4SLinus Torvalds /**************************************************************************
18391da177e4SLinus Torvalds **
18401da177e4SLinus Torvalds ** SBA initialization code (HW and SW)
18411da177e4SLinus Torvalds **
18421da177e4SLinus Torvalds ** o identify SBA chip itself
18431da177e4SLinus Torvalds ** o FIXME: initialize DMA hints for reasonable defaults
18441da177e4SLinus Torvalds **
18451da177e4SLinus Torvalds **************************************************************************/
18461da177e4SLinus Torvalds
18471da177e4SLinus Torvalds #ifdef CONFIG_PROC_FS
18481da177e4SLinus Torvalds static void *
ioc_start(struct seq_file * s,loff_t * pos)18491da177e4SLinus Torvalds ioc_start(struct seq_file *s, loff_t *pos)
18501da177e4SLinus Torvalds {
18511da177e4SLinus Torvalds struct ioc *ioc;
18521da177e4SLinus Torvalds loff_t n = *pos;
18531da177e4SLinus Torvalds
18541da177e4SLinus Torvalds for (ioc = ioc_list; ioc; ioc = ioc->next)
18551da177e4SLinus Torvalds if (!n--)
18561da177e4SLinus Torvalds return ioc;
18571da177e4SLinus Torvalds
18581da177e4SLinus Torvalds return NULL;
18591da177e4SLinus Torvalds }
18601da177e4SLinus Torvalds
18611da177e4SLinus Torvalds static void *
ioc_next(struct seq_file * s,void * v,loff_t * pos)18621da177e4SLinus Torvalds ioc_next(struct seq_file *s, void *v, loff_t *pos)
18631da177e4SLinus Torvalds {
18641da177e4SLinus Torvalds struct ioc *ioc = v;
18651da177e4SLinus Torvalds
18661da177e4SLinus Torvalds ++*pos;
18671da177e4SLinus Torvalds return ioc->next;
18681da177e4SLinus Torvalds }
18691da177e4SLinus Torvalds
18701da177e4SLinus Torvalds static void
ioc_stop(struct seq_file * s,void * v)18711da177e4SLinus Torvalds ioc_stop(struct seq_file *s, void *v)
18721da177e4SLinus Torvalds {
18731da177e4SLinus Torvalds }
18741da177e4SLinus Torvalds
18751da177e4SLinus Torvalds static int
ioc_show(struct seq_file * s,void * v)18761da177e4SLinus Torvalds ioc_show(struct seq_file *s, void *v)
18771da177e4SLinus Torvalds {
18781da177e4SLinus Torvalds struct ioc *ioc = v;
18791da177e4SLinus Torvalds unsigned long *res_ptr = (unsigned long *)ioc->res_map;
18801da177e4SLinus Torvalds int i, used = 0;
18811da177e4SLinus Torvalds
18821da177e4SLinus Torvalds seq_printf(s, "Hewlett Packard %s IOC rev %d.%d\n",
18831da177e4SLinus Torvalds ioc->name, ((ioc->rev >> 4) & 0xF), (ioc->rev & 0xF));
18841da177e4SLinus Torvalds #ifdef CONFIG_NUMA
188510ee3d70SBjorn Helgaas if (ioc->node != NUMA_NO_NODE)
18861da177e4SLinus Torvalds seq_printf(s, "NUMA node : %d\n", ioc->node);
18871da177e4SLinus Torvalds #endif
18881da177e4SLinus Torvalds seq_printf(s, "IOVA size : %ld MB\n", ((ioc->pdir_size >> 3) * iovp_size)/(1024*1024));
18891da177e4SLinus Torvalds seq_printf(s, "IOVA page size : %ld kb\n", iovp_size/1024);
18901da177e4SLinus Torvalds
18911da177e4SLinus Torvalds for (i = 0; i < (ioc->res_size / sizeof(unsigned long)); ++i, ++res_ptr)
18921da177e4SLinus Torvalds used += hweight64(*res_ptr);
18931da177e4SLinus Torvalds
18941da177e4SLinus Torvalds seq_printf(s, "PDIR size : %d entries\n", ioc->pdir_size >> 3);
18951da177e4SLinus Torvalds seq_printf(s, "PDIR used : %d entries\n", used);
18961da177e4SLinus Torvalds
18971da177e4SLinus Torvalds #ifdef PDIR_SEARCH_TIMING
18981da177e4SLinus Torvalds {
18991da177e4SLinus Torvalds unsigned long i = 0, avg = 0, min, max;
19001da177e4SLinus Torvalds min = max = ioc->avg_search[0];
19011da177e4SLinus Torvalds for (i = 0; i < SBA_SEARCH_SAMPLE; i++) {
19021da177e4SLinus Torvalds avg += ioc->avg_search[i];
19031da177e4SLinus Torvalds if (ioc->avg_search[i] > max) max = ioc->avg_search[i];
19041da177e4SLinus Torvalds if (ioc->avg_search[i] < min) min = ioc->avg_search[i];
19051da177e4SLinus Torvalds }
19061da177e4SLinus Torvalds avg /= SBA_SEARCH_SAMPLE;
19071da177e4SLinus Torvalds seq_printf(s, "Bitmap search : %ld/%ld/%ld (min/avg/max CPU Cycles/IOVA page)\n",
19081da177e4SLinus Torvalds min, avg, max);
19091da177e4SLinus Torvalds }
19101da177e4SLinus Torvalds #endif
19111da177e4SLinus Torvalds #ifndef ALLOW_IOV_BYPASS
19121da177e4SLinus Torvalds seq_printf(s, "IOVA bypass disabled\n");
19131da177e4SLinus Torvalds #endif
19141da177e4SLinus Torvalds return 0;
19151da177e4SLinus Torvalds }
19161da177e4SLinus Torvalds
1917a23fe55eSJan Engelhardt static const struct seq_operations ioc_seq_ops = {
19181da177e4SLinus Torvalds .start = ioc_start,
19191da177e4SLinus Torvalds .next = ioc_next,
19201da177e4SLinus Torvalds .stop = ioc_stop,
19211da177e4SLinus Torvalds .show = ioc_show
19221da177e4SLinus Torvalds };
19231da177e4SLinus Torvalds
19241da177e4SLinus Torvalds static void __init
ioc_proc_init(void)19251da177e4SLinus Torvalds ioc_proc_init(void)
19261da177e4SLinus Torvalds {
1927e2363768SDenis V. Lunev struct proc_dir_entry *dir;
19281da177e4SLinus Torvalds
19291da177e4SLinus Torvalds dir = proc_mkdir("bus/mckinley", NULL);
19301da177e4SLinus Torvalds if (!dir)
19311da177e4SLinus Torvalds return;
19321da177e4SLinus Torvalds
1933fddda2b7SChristoph Hellwig proc_create_seq(ioc_list->name, 0, dir, &ioc_seq_ops);
19341da177e4SLinus Torvalds }
19351da177e4SLinus Torvalds #endif
19361da177e4SLinus Torvalds
19371da177e4SLinus Torvalds static void
sba_connect_bus(struct pci_bus * bus)19381da177e4SLinus Torvalds sba_connect_bus(struct pci_bus *bus)
19391da177e4SLinus Torvalds {
19401da177e4SLinus Torvalds acpi_handle handle, parent;
19411da177e4SLinus Torvalds acpi_status status;
19421da177e4SLinus Torvalds struct ioc *ioc;
19431da177e4SLinus Torvalds
19441da177e4SLinus Torvalds if (!PCI_CONTROLLER(bus))
19451da177e4SLinus Torvalds panic(PFX "no sysdata on bus %d!\n", bus->number);
19461da177e4SLinus Torvalds
19471da177e4SLinus Torvalds if (PCI_CONTROLLER(bus)->iommu)
19481da177e4SLinus Torvalds return;
19491da177e4SLinus Torvalds
19507b199811SRafael J. Wysocki handle = acpi_device_handle(PCI_CONTROLLER(bus)->companion);
19511da177e4SLinus Torvalds if (!handle)
19521da177e4SLinus Torvalds return;
19531da177e4SLinus Torvalds
19541da177e4SLinus Torvalds /*
19551da177e4SLinus Torvalds * The IOC scope encloses PCI root bridges in the ACPI
19561da177e4SLinus Torvalds * namespace, so work our way out until we find an IOC we
19571da177e4SLinus Torvalds * claimed previously.
19581da177e4SLinus Torvalds */
19591da177e4SLinus Torvalds do {
19601da177e4SLinus Torvalds for (ioc = ioc_list; ioc; ioc = ioc->next)
19611da177e4SLinus Torvalds if (ioc->handle == handle) {
19621da177e4SLinus Torvalds PCI_CONTROLLER(bus)->iommu = ioc;
19631da177e4SLinus Torvalds return;
19641da177e4SLinus Torvalds }
19651da177e4SLinus Torvalds
19661da177e4SLinus Torvalds status = acpi_get_parent(handle, &parent);
19671da177e4SLinus Torvalds handle = parent;
19681da177e4SLinus Torvalds } while (ACPI_SUCCESS(status));
19691da177e4SLinus Torvalds
19701da177e4SLinus Torvalds printk(KERN_WARNING "No IOC for PCI Bus %04x:%02x in ACPI\n", pci_domain_nr(bus), bus->number);
19711da177e4SLinus Torvalds }
19721da177e4SLinus Torvalds
19731da177e4SLinus Torvalds static void __init
sba_map_ioc_to_node(struct ioc * ioc,acpi_handle handle)19741da177e4SLinus Torvalds sba_map_ioc_to_node(struct ioc *ioc, acpi_handle handle)
19751da177e4SLinus Torvalds {
1976b1e9cee7SBjorn Helgaas #ifdef CONFIG_NUMA
19771da177e4SLinus Torvalds unsigned int node;
19781da177e4SLinus Torvalds
1979b1e9cee7SBjorn Helgaas node = acpi_get_node(handle);
1980b1e9cee7SBjorn Helgaas if (node != NUMA_NO_NODE && !node_online(node))
1981b1e9cee7SBjorn Helgaas node = NUMA_NO_NODE;
19821da177e4SLinus Torvalds
19831da177e4SLinus Torvalds ioc->node = node;
19841da177e4SLinus Torvalds #endif
1985b1e9cee7SBjorn Helgaas }
19861da177e4SLinus Torvalds
acpi_sba_ioc_add(struct ioc * ioc)19872edd73a4STony Luck static void __init acpi_sba_ioc_add(struct ioc *ioc)
19881da177e4SLinus Torvalds {
198912e27b11SRafael J. Wysocki acpi_handle handle = ioc->handle;
19901da177e4SLinus Torvalds acpi_status status;
19911da177e4SLinus Torvalds u64 hpa, length;
199280aa9bf0SJoe Perches struct acpi_device_info *adi;
19931da177e4SLinus Torvalds
199412e27b11SRafael J. Wysocki ioc_found = ioc->next;
199512e27b11SRafael J. Wysocki status = hp_acpi_csr_space(handle, &hpa, &length);
19961da177e4SLinus Torvalds if (ACPI_FAILURE(status))
199712e27b11SRafael J. Wysocki goto err;
19981da177e4SLinus Torvalds
199912e27b11SRafael J. Wysocki status = acpi_get_object_info(handle, &adi);
20001da177e4SLinus Torvalds if (ACPI_FAILURE(status))
200112e27b11SRafael J. Wysocki goto err;
20021da177e4SLinus Torvalds
20031da177e4SLinus Torvalds /*
20041da177e4SLinus Torvalds * For HWP0001, only SBA appears in ACPI namespace. It encloses the PCI
20051da177e4SLinus Torvalds * root bridges, and its CSR space includes the IOC function.
20061da177e4SLinus Torvalds */
200780aa9bf0SJoe Perches if (strncmp("HWP0001", adi->hardware_id.string, 7) == 0) {
20081da177e4SLinus Torvalds hpa += ZX1_IOC_OFFSET;
20091da177e4SLinus Torvalds /* zx1 based systems default to kernel page size iommu pages */
20101da177e4SLinus Torvalds if (!iovp_shift)
20111da177e4SLinus Torvalds iovp_shift = min(PAGE_SHIFT, 16);
20121da177e4SLinus Torvalds }
201380aa9bf0SJoe Perches kfree(adi);
20141da177e4SLinus Torvalds
20151da177e4SLinus Torvalds /*
20161da177e4SLinus Torvalds * default anything not caught above or specified on cmdline to 4k
20171da177e4SLinus Torvalds * iommu page size
20181da177e4SLinus Torvalds */
20191da177e4SLinus Torvalds if (!iovp_shift)
20201da177e4SLinus Torvalds iovp_shift = 12;
20211da177e4SLinus Torvalds
202212e27b11SRafael J. Wysocki ioc_init(hpa, ioc);
20231da177e4SLinus Torvalds /* setup NUMA node association */
202412e27b11SRafael J. Wysocki sba_map_ioc_to_node(ioc, handle);
202512e27b11SRafael J. Wysocki return;
202612e27b11SRafael J. Wysocki
202712e27b11SRafael J. Wysocki err:
202812e27b11SRafael J. Wysocki kfree(ioc);
20291da177e4SLinus Torvalds }
20301da177e4SLinus Torvalds
20317091138fSThomas Renninger static const struct acpi_device_id hp_ioc_iommu_device_ids[] = {
20327091138fSThomas Renninger {"HWP0001", 0},
20337091138fSThomas Renninger {"HWP0004", 0},
20347091138fSThomas Renninger {"", 0},
20357091138fSThomas Renninger };
203612e27b11SRafael J. Wysocki
acpi_sba_ioc_attach(struct acpi_device * device,const struct acpi_device_id * not_used)203712e27b11SRafael J. Wysocki static int acpi_sba_ioc_attach(struct acpi_device *device,
203812e27b11SRafael J. Wysocki const struct acpi_device_id *not_used)
203912e27b11SRafael J. Wysocki {
204012e27b11SRafael J. Wysocki struct ioc *ioc;
204112e27b11SRafael J. Wysocki
204212e27b11SRafael J. Wysocki ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
204312e27b11SRafael J. Wysocki if (!ioc)
204412e27b11SRafael J. Wysocki return -ENOMEM;
204512e27b11SRafael J. Wysocki
204612e27b11SRafael J. Wysocki ioc->next = ioc_found;
204712e27b11SRafael J. Wysocki ioc_found = ioc;
204812e27b11SRafael J. Wysocki ioc->handle = device->handle;
204912e27b11SRafael J. Wysocki return 1;
205012e27b11SRafael J. Wysocki }
205112e27b11SRafael J. Wysocki
205212e27b11SRafael J. Wysocki
205366345d5fSRafael J. Wysocki static struct acpi_scan_handler acpi_sba_ioc_handler = {
20547091138fSThomas Renninger .ids = hp_ioc_iommu_device_ids,
205512e27b11SRafael J. Wysocki .attach = acpi_sba_ioc_attach,
20561da177e4SLinus Torvalds };
20571da177e4SLinus Torvalds
acpi_sba_ioc_init_acpi(void)205866345d5fSRafael J. Wysocki static int __init acpi_sba_ioc_init_acpi(void)
205966345d5fSRafael J. Wysocki {
206066345d5fSRafael J. Wysocki return acpi_scan_add_handler(&acpi_sba_ioc_handler);
206166345d5fSRafael J. Wysocki }
206266345d5fSRafael J. Wysocki /* This has to run before acpi_scan_init(). */
206366345d5fSRafael J. Wysocki arch_initcall(acpi_sba_ioc_init_acpi);
206466345d5fSRafael J. Wysocki
sba_dma_supported(struct device * dev,u64 mask)2065974f83ecSChristoph Hellwig static int sba_dma_supported (struct device *dev, u64 mask)
2066974f83ecSChristoph Hellwig {
2067974f83ecSChristoph Hellwig /* make sure it's at least 32bit capable */
2068974f83ecSChristoph Hellwig return ((mask & 0xFFFFFFFFUL) == 0xFFFFFFFFUL);
2069974f83ecSChristoph Hellwig }
2070974f83ecSChristoph Hellwig
2071974f83ecSChristoph Hellwig static const struct dma_map_ops sba_dma_ops = {
2072974f83ecSChristoph Hellwig .alloc = sba_alloc_coherent,
2073974f83ecSChristoph Hellwig .free = sba_free_coherent,
2074974f83ecSChristoph Hellwig .map_page = sba_map_page,
2075974f83ecSChristoph Hellwig .unmap_page = sba_unmap_page,
2076974f83ecSChristoph Hellwig .map_sg = sba_map_sg_attrs,
2077974f83ecSChristoph Hellwig .unmap_sg = sba_unmap_sg_attrs,
2078974f83ecSChristoph Hellwig .dma_supported = sba_dma_supported,
2079671df189SLinus Torvalds .mmap = dma_common_mmap,
2080671df189SLinus Torvalds .get_sgtable = dma_common_get_sgtable,
2081efa70f2fSChristoph Hellwig .alloc_pages = dma_common_alloc_pages,
2082efa70f2fSChristoph Hellwig .free_pages = dma_common_free_pages,
2083974f83ecSChristoph Hellwig };
2084974f83ecSChristoph Hellwig
20851da177e4SLinus Torvalds static int __init
sba_init(void)20861da177e4SLinus Torvalds sba_init(void)
20871da177e4SLinus Torvalds {
2088974f83ecSChristoph Hellwig /*
2089974f83ecSChristoph Hellwig * If we are booting a kdump kernel, the sba_iommu will cause devices
2090974f83ecSChristoph Hellwig * that were not shutdown properly to MCA as soon as they are turned
2091974f83ecSChristoph Hellwig * back on. Our only option for a successful kdump kernel boot is to
2092974f83ecSChristoph Hellwig * use swiotlb.
209351b58e3eSTerry Loftin */
2094974f83ecSChristoph Hellwig if (is_kdump_kernel())
209551b58e3eSTerry Loftin return 0;
209651b58e3eSTerry Loftin
209766345d5fSRafael J. Wysocki /*
209812e27b11SRafael J. Wysocki * ioc_found should be populated by the acpi_sba_ioc_handler's .attach()
209966345d5fSRafael J. Wysocki * routine, but that only happens if acpi_scan_init() has already run.
210066345d5fSRafael J. Wysocki */
210112e27b11SRafael J. Wysocki while (ioc_found)
210212e27b11SRafael J. Wysocki acpi_sba_ioc_add(ioc_found);
210312e27b11SRafael J. Wysocki
2104974f83ecSChristoph Hellwig if (!ioc_list)
21050b9afedeSAlex Williamson return 0;
21060b9afedeSAlex Williamson
21071da177e4SLinus Torvalds {
21081da177e4SLinus Torvalds struct pci_bus *b = NULL;
21091da177e4SLinus Torvalds while ((b = pci_find_next_bus(b)) != NULL)
21101da177e4SLinus Torvalds sba_connect_bus(b);
21111da177e4SLinus Torvalds }
21121da177e4SLinus Torvalds
2113974f83ecSChristoph Hellwig /* no need for swiotlb with the iommu */
2114974f83ecSChristoph Hellwig swiotlb_exit();
2115974f83ecSChristoph Hellwig dma_ops = &sba_dma_ops;
2116974f83ecSChristoph Hellwig
21171da177e4SLinus Torvalds #ifdef CONFIG_PROC_FS
21181da177e4SLinus Torvalds ioc_proc_init();
21191da177e4SLinus Torvalds #endif
21201da177e4SLinus Torvalds return 0;
21211da177e4SLinus Torvalds }
21221da177e4SLinus Torvalds
21231da177e4SLinus Torvalds subsys_initcall(sba_init); /* must be initialized after ACPI etc., but before any drivers... */
21241da177e4SLinus Torvalds
21251da177e4SLinus Torvalds static int __init
nosbagart(char * str)21261da177e4SLinus Torvalds nosbagart(char *str)
21271da177e4SLinus Torvalds {
21281da177e4SLinus Torvalds reserve_sba_gart = 0;
21291da177e4SLinus Torvalds return 1;
21301da177e4SLinus Torvalds }
21311da177e4SLinus Torvalds
21321da177e4SLinus Torvalds __setup("nosbagart", nosbagart);
21331da177e4SLinus Torvalds
21341da177e4SLinus Torvalds static int __init
sba_page_override(char * str)21351da177e4SLinus Torvalds sba_page_override(char *str)
21361da177e4SLinus Torvalds {
21371da177e4SLinus Torvalds unsigned long page_size;
21381da177e4SLinus Torvalds
21391da177e4SLinus Torvalds page_size = memparse(str, &str);
21401da177e4SLinus Torvalds switch (page_size) {
21411da177e4SLinus Torvalds case 4096:
21421da177e4SLinus Torvalds case 8192:
21431da177e4SLinus Torvalds case 16384:
21441da177e4SLinus Torvalds case 65536:
21451da177e4SLinus Torvalds iovp_shift = ffs(page_size) - 1;
21461da177e4SLinus Torvalds break;
21471da177e4SLinus Torvalds default:
21481da177e4SLinus Torvalds printk("%s: unknown/unsupported iommu page size %ld\n",
2149d4ed8084SHarvey Harrison __func__, page_size);
21501da177e4SLinus Torvalds }
21511da177e4SLinus Torvalds
21521da177e4SLinus Torvalds return 1;
21531da177e4SLinus Torvalds }
21541da177e4SLinus Torvalds
21551da177e4SLinus Torvalds __setup("sbapagesize=",sba_page_override);
2156