15933f6d2SKuninori Morimoto // SPDX-License-Identifier: GPL-2.0 2e7cc9a73SMagnus Damm /* 3e7cc9a73SMagnus Damm * Trapped io support 4e7cc9a73SMagnus Damm * 5e7cc9a73SMagnus Damm * Copyright (C) 2008 Magnus Damm 6e7cc9a73SMagnus Damm * 7e7cc9a73SMagnus Damm * Intercept io operations by trapping. 8e7cc9a73SMagnus Damm */ 9e7cc9a73SMagnus Damm #include <linux/kernel.h> 10e7cc9a73SMagnus Damm #include <linux/mm.h> 11e7cc9a73SMagnus Damm #include <linux/bitops.h> 12e7cc9a73SMagnus Damm #include <linux/vmalloc.h> 13ecc14e8cSPaul Mundt #include <linux/module.h> 14eeee7853SPaul Mundt #include <linux/init.h> 15e7cc9a73SMagnus Damm #include <asm/mmu_context.h> 167c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 17e7cc9a73SMagnus Damm #include <asm/io.h> 18e7cc9a73SMagnus Damm #include <asm/io_trapped.h> 19e7cc9a73SMagnus Damm 20e7cc9a73SMagnus Damm #define TRAPPED_PAGES_MAX 16 21e7cc9a73SMagnus Damm 22ce816fa8SUwe Kleine-König #ifdef CONFIG_HAS_IOPORT_MAP 23e7cc9a73SMagnus Damm LIST_HEAD(trapped_io); 24ecc14e8cSPaul Mundt EXPORT_SYMBOL_GPL(trapped_io); 25e7cc9a73SMagnus Damm #endif 26e7cc9a73SMagnus Damm #ifdef CONFIG_HAS_IOMEM 27e7cc9a73SMagnus Damm LIST_HEAD(trapped_mem); 28ecc14e8cSPaul Mundt EXPORT_SYMBOL_GPL(trapped_mem); 29e7cc9a73SMagnus Damm #endif 30e7cc9a73SMagnus Damm static DEFINE_SPINLOCK(trapped_lock); 31e7cc9a73SMagnus Damm 32eeee7853SPaul Mundt static int trapped_io_disable __read_mostly; 33eeee7853SPaul Mundt 34eeee7853SPaul Mundt static int __init trapped_io_setup(char *__unused) 35eeee7853SPaul Mundt { 36eeee7853SPaul Mundt trapped_io_disable = 1; 37eeee7853SPaul Mundt return 1; 38eeee7853SPaul Mundt } 39eeee7853SPaul Mundt __setup("noiotrap", trapped_io_setup); 40eeee7853SPaul Mundt 41b2839ed8SPaul Mundt int register_trapped_io(struct trapped_io *tiop) 42e7cc9a73SMagnus Damm { 43e7cc9a73SMagnus Damm struct resource *res; 44e7cc9a73SMagnus Damm unsigned long len = 0, flags = 0; 45e7cc9a73SMagnus Damm struct page *pages[TRAPPED_PAGES_MAX]; 46e7cc9a73SMagnus Damm int k, n; 47e7cc9a73SMagnus Damm 48eeee7853SPaul Mundt if (unlikely(trapped_io_disable)) 49eeee7853SPaul Mundt return 0; 50eeee7853SPaul Mundt 51e7cc9a73SMagnus Damm /* structure must be page aligned */ 52e7cc9a73SMagnus Damm if ((unsigned long)tiop & (PAGE_SIZE - 1)) 53e7cc9a73SMagnus Damm goto bad; 54e7cc9a73SMagnus Damm 55e7cc9a73SMagnus Damm for (k = 0; k < tiop->num_resources; k++) { 56e7cc9a73SMagnus Damm res = tiop->resource + k; 5728f65c11SJoe Perches len += roundup(resource_size(res), PAGE_SIZE); 58e7cc9a73SMagnus Damm flags |= res->flags; 59e7cc9a73SMagnus Damm } 60e7cc9a73SMagnus Damm 61e7cc9a73SMagnus Damm /* support IORESOURCE_IO _or_ MEM, not both */ 62e7cc9a73SMagnus Damm if (hweight_long(flags) != 1) 63e7cc9a73SMagnus Damm goto bad; 64e7cc9a73SMagnus Damm 65e7cc9a73SMagnus Damm n = len >> PAGE_SHIFT; 66e7cc9a73SMagnus Damm 67e7cc9a73SMagnus Damm if (n >= TRAPPED_PAGES_MAX) 68e7cc9a73SMagnus Damm goto bad; 69e7cc9a73SMagnus Damm 70e7cc9a73SMagnus Damm for (k = 0; k < n; k++) 71e7cc9a73SMagnus Damm pages[k] = virt_to_page(tiop); 72e7cc9a73SMagnus Damm 73e7cc9a73SMagnus Damm tiop->virt_base = vmap(pages, n, VM_MAP, PAGE_NONE); 74e7cc9a73SMagnus Damm if (!tiop->virt_base) 75e7cc9a73SMagnus Damm goto bad; 76e7cc9a73SMagnus Damm 77e7cc9a73SMagnus Damm len = 0; 78e7cc9a73SMagnus Damm for (k = 0; k < tiop->num_resources; k++) { 79e7cc9a73SMagnus Damm res = tiop->resource + k; 80e7cc9a73SMagnus Damm pr_info("trapped io 0x%08lx overrides %s 0x%08lx\n", 81e7cc9a73SMagnus Damm (unsigned long)(tiop->virt_base + len), 82e7cc9a73SMagnus Damm res->flags & IORESOURCE_IO ? "io" : "mmio", 83e7cc9a73SMagnus Damm (unsigned long)res->start); 8428f65c11SJoe Perches len += roundup(resource_size(res), PAGE_SIZE); 85e7cc9a73SMagnus Damm } 86e7cc9a73SMagnus Damm 87e7cc9a73SMagnus Damm tiop->magic = IO_TRAPPED_MAGIC; 88e7cc9a73SMagnus Damm INIT_LIST_HEAD(&tiop->list); 89e7cc9a73SMagnus Damm spin_lock_irq(&trapped_lock); 90ce816fa8SUwe Kleine-König #ifdef CONFIG_HAS_IOPORT_MAP 91e7cc9a73SMagnus Damm if (flags & IORESOURCE_IO) 92e7cc9a73SMagnus Damm list_add(&tiop->list, &trapped_io); 9386e4dd5aSPaul Mundt #endif 9486e4dd5aSPaul Mundt #ifdef CONFIG_HAS_IOMEM 95e7cc9a73SMagnus Damm if (flags & IORESOURCE_MEM) 96e7cc9a73SMagnus Damm list_add(&tiop->list, &trapped_mem); 9786e4dd5aSPaul Mundt #endif 98e7cc9a73SMagnus Damm spin_unlock_irq(&trapped_lock); 99e7cc9a73SMagnus Damm 100e7cc9a73SMagnus Damm return 0; 101e7cc9a73SMagnus Damm bad: 1026d80f20cSKefeng Wang pr_warn("unable to install trapped io filter\n"); 103e7cc9a73SMagnus Damm return -1; 104e7cc9a73SMagnus Damm } 105e7cc9a73SMagnus Damm 106e7cc9a73SMagnus Damm void __iomem *match_trapped_io_handler(struct list_head *list, 107e7cc9a73SMagnus Damm unsigned long offset, 108e7cc9a73SMagnus Damm unsigned long size) 109e7cc9a73SMagnus Damm { 110e7cc9a73SMagnus Damm unsigned long voffs; 111e7cc9a73SMagnus Damm struct trapped_io *tiop; 112e7cc9a73SMagnus Damm struct resource *res; 113e7cc9a73SMagnus Damm int k, len; 114fd78a76aSStuart Menefy unsigned long flags; 115e7cc9a73SMagnus Damm 116fd78a76aSStuart Menefy spin_lock_irqsave(&trapped_lock, flags); 117e7cc9a73SMagnus Damm list_for_each_entry(tiop, list, list) { 118e7cc9a73SMagnus Damm voffs = 0; 119e7cc9a73SMagnus Damm for (k = 0; k < tiop->num_resources; k++) { 120e7cc9a73SMagnus Damm res = tiop->resource + k; 121e7cc9a73SMagnus Damm if (res->start == offset) { 122fd78a76aSStuart Menefy spin_unlock_irqrestore(&trapped_lock, flags); 123e7cc9a73SMagnus Damm return tiop->virt_base + voffs; 124e7cc9a73SMagnus Damm } 125e7cc9a73SMagnus Damm 12628f65c11SJoe Perches len = resource_size(res); 127e7cc9a73SMagnus Damm voffs += roundup(len, PAGE_SIZE); 128e7cc9a73SMagnus Damm } 129e7cc9a73SMagnus Damm } 130fd78a76aSStuart Menefy spin_unlock_irqrestore(&trapped_lock, flags); 131e7cc9a73SMagnus Damm return NULL; 132e7cc9a73SMagnus Damm } 133e7cc9a73SMagnus Damm 134e7cc9a73SMagnus Damm static struct trapped_io *lookup_tiop(unsigned long address) 135e7cc9a73SMagnus Damm { 136e7cc9a73SMagnus Damm pgd_t *pgd_k; 137874e2cc1SMike Rapoport p4d_t *p4d_k; 138e7cc9a73SMagnus Damm pud_t *pud_k; 139e7cc9a73SMagnus Damm pmd_t *pmd_k; 140e7cc9a73SMagnus Damm pte_t *pte_k; 141e7cc9a73SMagnus Damm pte_t entry; 142e7cc9a73SMagnus Damm 143e7cc9a73SMagnus Damm pgd_k = swapper_pg_dir + pgd_index(address); 144e7cc9a73SMagnus Damm if (!pgd_present(*pgd_k)) 145e7cc9a73SMagnus Damm return NULL; 146e7cc9a73SMagnus Damm 147874e2cc1SMike Rapoport p4d_k = p4d_offset(pgd_k, address); 148874e2cc1SMike Rapoport if (!p4d_present(*p4d_k)) 149874e2cc1SMike Rapoport return NULL; 150874e2cc1SMike Rapoport 151874e2cc1SMike Rapoport pud_k = pud_offset(p4d_k, address); 152e7cc9a73SMagnus Damm if (!pud_present(*pud_k)) 153e7cc9a73SMagnus Damm return NULL; 154e7cc9a73SMagnus Damm 155e7cc9a73SMagnus Damm pmd_k = pmd_offset(pud_k, address); 156e7cc9a73SMagnus Damm if (!pmd_present(*pmd_k)) 157e7cc9a73SMagnus Damm return NULL; 158e7cc9a73SMagnus Damm 159e7cc9a73SMagnus Damm pte_k = pte_offset_kernel(pmd_k, address); 160e7cc9a73SMagnus Damm entry = *pte_k; 161e7cc9a73SMagnus Damm 162e7cc9a73SMagnus Damm return pfn_to_kaddr(pte_pfn(entry)); 163e7cc9a73SMagnus Damm } 164e7cc9a73SMagnus Damm 165e7cc9a73SMagnus Damm static unsigned long lookup_address(struct trapped_io *tiop, 166e7cc9a73SMagnus Damm unsigned long address) 167e7cc9a73SMagnus Damm { 168e7cc9a73SMagnus Damm struct resource *res; 169e7cc9a73SMagnus Damm unsigned long vaddr = (unsigned long)tiop->virt_base; 170e7cc9a73SMagnus Damm unsigned long len; 171e7cc9a73SMagnus Damm int k; 172e7cc9a73SMagnus Damm 173e7cc9a73SMagnus Damm for (k = 0; k < tiop->num_resources; k++) { 174e7cc9a73SMagnus Damm res = tiop->resource + k; 17528f65c11SJoe Perches len = roundup(resource_size(res), PAGE_SIZE); 176e7cc9a73SMagnus Damm if (address < (vaddr + len)) 177e7cc9a73SMagnus Damm return res->start + (address - vaddr); 178e7cc9a73SMagnus Damm vaddr += len; 179e7cc9a73SMagnus Damm } 180e7cc9a73SMagnus Damm return 0; 181e7cc9a73SMagnus Damm } 182e7cc9a73SMagnus Damm 183e7cc9a73SMagnus Damm static unsigned long long copy_word(unsigned long src_addr, int src_len, 184e7cc9a73SMagnus Damm unsigned long dst_addr, int dst_len) 185e7cc9a73SMagnus Damm { 186e7cc9a73SMagnus Damm unsigned long long tmp = 0; 187e7cc9a73SMagnus Damm 188e7cc9a73SMagnus Damm switch (src_len) { 189e7cc9a73SMagnus Damm case 1: 1909d56dd3bSPaul Mundt tmp = __raw_readb(src_addr); 191e7cc9a73SMagnus Damm break; 192e7cc9a73SMagnus Damm case 2: 1939d56dd3bSPaul Mundt tmp = __raw_readw(src_addr); 194e7cc9a73SMagnus Damm break; 195e7cc9a73SMagnus Damm case 4: 1969d56dd3bSPaul Mundt tmp = __raw_readl(src_addr); 197e7cc9a73SMagnus Damm break; 198e7cc9a73SMagnus Damm case 8: 1999d56dd3bSPaul Mundt tmp = __raw_readq(src_addr); 200e7cc9a73SMagnus Damm break; 201e7cc9a73SMagnus Damm } 202e7cc9a73SMagnus Damm 203e7cc9a73SMagnus Damm switch (dst_len) { 204e7cc9a73SMagnus Damm case 1: 2059d56dd3bSPaul Mundt __raw_writeb(tmp, dst_addr); 206e7cc9a73SMagnus Damm break; 207e7cc9a73SMagnus Damm case 2: 2089d56dd3bSPaul Mundt __raw_writew(tmp, dst_addr); 209e7cc9a73SMagnus Damm break; 210e7cc9a73SMagnus Damm case 4: 2119d56dd3bSPaul Mundt __raw_writel(tmp, dst_addr); 212e7cc9a73SMagnus Damm break; 213e7cc9a73SMagnus Damm case 8: 2149d56dd3bSPaul Mundt __raw_writeq(tmp, dst_addr); 215e7cc9a73SMagnus Damm break; 216e7cc9a73SMagnus Damm } 217e7cc9a73SMagnus Damm 218e7cc9a73SMagnus Damm return tmp; 219e7cc9a73SMagnus Damm } 220e7cc9a73SMagnus Damm 221e7cc9a73SMagnus Damm static unsigned long from_device(void *dst, const void *src, unsigned long cnt) 222e7cc9a73SMagnus Damm { 223e7cc9a73SMagnus Damm struct trapped_io *tiop; 224e7cc9a73SMagnus Damm unsigned long src_addr = (unsigned long)src; 225e7cc9a73SMagnus Damm unsigned long long tmp; 226e7cc9a73SMagnus Damm 227e7cc9a73SMagnus Damm pr_debug("trapped io read 0x%08lx (%ld)\n", src_addr, cnt); 228e7cc9a73SMagnus Damm tiop = lookup_tiop(src_addr); 229e7cc9a73SMagnus Damm WARN_ON(!tiop || (tiop->magic != IO_TRAPPED_MAGIC)); 230e7cc9a73SMagnus Damm 231e7cc9a73SMagnus Damm src_addr = lookup_address(tiop, src_addr); 232e7cc9a73SMagnus Damm if (!src_addr) 233e7cc9a73SMagnus Damm return cnt; 234e7cc9a73SMagnus Damm 235f1cdd63fSPaul Mundt tmp = copy_word(src_addr, 236f1cdd63fSPaul Mundt max_t(unsigned long, cnt, 237f1cdd63fSPaul Mundt (tiop->minimum_bus_width / 8)), 238e7cc9a73SMagnus Damm (unsigned long)dst, cnt); 239e7cc9a73SMagnus Damm 240e7cc9a73SMagnus Damm pr_debug("trapped io read 0x%08lx -> 0x%08llx\n", src_addr, tmp); 241e7cc9a73SMagnus Damm return 0; 242e7cc9a73SMagnus Damm } 243e7cc9a73SMagnus Damm 244e7cc9a73SMagnus Damm static unsigned long to_device(void *dst, const void *src, unsigned long cnt) 245e7cc9a73SMagnus Damm { 246e7cc9a73SMagnus Damm struct trapped_io *tiop; 247e7cc9a73SMagnus Damm unsigned long dst_addr = (unsigned long)dst; 248e7cc9a73SMagnus Damm unsigned long long tmp; 249e7cc9a73SMagnus Damm 250e7cc9a73SMagnus Damm pr_debug("trapped io write 0x%08lx (%ld)\n", dst_addr, cnt); 251e7cc9a73SMagnus Damm tiop = lookup_tiop(dst_addr); 252e7cc9a73SMagnus Damm WARN_ON(!tiop || (tiop->magic != IO_TRAPPED_MAGIC)); 253e7cc9a73SMagnus Damm 254e7cc9a73SMagnus Damm dst_addr = lookup_address(tiop, dst_addr); 255e7cc9a73SMagnus Damm if (!dst_addr) 256e7cc9a73SMagnus Damm return cnt; 257e7cc9a73SMagnus Damm 258e7cc9a73SMagnus Damm tmp = copy_word((unsigned long)src, cnt, 259f1cdd63fSPaul Mundt dst_addr, max_t(unsigned long, cnt, 260f1cdd63fSPaul Mundt (tiop->minimum_bus_width / 8))); 261e7cc9a73SMagnus Damm 262e7cc9a73SMagnus Damm pr_debug("trapped io write 0x%08lx -> 0x%08llx\n", dst_addr, tmp); 263e7cc9a73SMagnus Damm return 0; 264e7cc9a73SMagnus Damm } 265e7cc9a73SMagnus Damm 266e7cc9a73SMagnus Damm static struct mem_access trapped_io_access = { 267e7cc9a73SMagnus Damm from_device, 268e7cc9a73SMagnus Damm to_device, 269e7cc9a73SMagnus Damm }; 270e7cc9a73SMagnus Damm 271e7cc9a73SMagnus Damm int handle_trapped_io(struct pt_regs *regs, unsigned long address) 272e7cc9a73SMagnus Damm { 2732bcfffa4SPaul Mundt insn_size_t instruction; 274e7cc9a73SMagnus Damm int tmp; 275e7cc9a73SMagnus Damm 27608b36c4aSPaul Mundt if (trapped_io_disable) 27708b36c4aSPaul Mundt return 0; 278e7cc9a73SMagnus Damm if (!lookup_tiop(address)) 279e7cc9a73SMagnus Damm return 0; 280e7cc9a73SMagnus Damm 281e7cc9a73SMagnus Damm WARN_ON(user_mode(regs)); 282e7cc9a73SMagnus Damm 283*75d4d295SArnd Bergmann if (copy_from_kernel_nofault(&instruction, (void *)(regs->pc), 284e7cc9a73SMagnus Damm sizeof(instruction))) { 285e7cc9a73SMagnus Damm return 0; 286e7cc9a73SMagnus Damm } 287e7cc9a73SMagnus Damm 2884aa5ac4eSMatt Fleming tmp = handle_unaligned_access(instruction, regs, 289ace2dc7dSPaul Mundt &trapped_io_access, 1, address); 290e7cc9a73SMagnus Damm return tmp == 0; 291e7cc9a73SMagnus Damm } 292