15933f6d2SKuninori Morimoto // SPDX-License-Identifier: GPL-2.0 2e7cc9a73SMagnus Damm /* 3e7cc9a73SMagnus Damm * Trapped io support 4e7cc9a73SMagnus Damm * 5e7cc9a73SMagnus Damm * Copyright (C) 2008 Magnus Damm 6e7cc9a73SMagnus Damm * 7e7cc9a73SMagnus Damm * Intercept io operations by trapping. 8e7cc9a73SMagnus Damm */ 9e7cc9a73SMagnus Damm #include <linux/kernel.h> 10e7cc9a73SMagnus Damm #include <linux/mm.h> 11e7cc9a73SMagnus Damm #include <linux/bitops.h> 12e7cc9a73SMagnus Damm #include <linux/vmalloc.h> 13ecc14e8cSPaul Mundt #include <linux/module.h> 14eeee7853SPaul Mundt #include <linux/init.h> 15e7cc9a73SMagnus Damm #include <asm/mmu_context.h> 167c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 17e7cc9a73SMagnus Damm #include <asm/io.h> 18e7cc9a73SMagnus Damm #include <asm/io_trapped.h> 19e7cc9a73SMagnus Damm 20e7cc9a73SMagnus Damm #define TRAPPED_PAGES_MAX 16 21e7cc9a73SMagnus Damm 22ce816fa8SUwe Kleine-König #ifdef CONFIG_HAS_IOPORT_MAP 23e7cc9a73SMagnus Damm LIST_HEAD(trapped_io); 24ecc14e8cSPaul Mundt EXPORT_SYMBOL_GPL(trapped_io); 25e7cc9a73SMagnus Damm #endif 26e7cc9a73SMagnus Damm #ifdef CONFIG_HAS_IOMEM 27e7cc9a73SMagnus Damm LIST_HEAD(trapped_mem); 28ecc14e8cSPaul Mundt EXPORT_SYMBOL_GPL(trapped_mem); 29e7cc9a73SMagnus Damm #endif 30e7cc9a73SMagnus Damm static DEFINE_SPINLOCK(trapped_lock); 31e7cc9a73SMagnus Damm 32eeee7853SPaul Mundt static int trapped_io_disable __read_mostly; 33eeee7853SPaul Mundt 34eeee7853SPaul Mundt static int __init trapped_io_setup(char *__unused) 35eeee7853SPaul Mundt { 36eeee7853SPaul Mundt trapped_io_disable = 1; 37eeee7853SPaul Mundt return 1; 38eeee7853SPaul Mundt } 39eeee7853SPaul Mundt __setup("noiotrap", trapped_io_setup); 40eeee7853SPaul Mundt 41b2839ed8SPaul Mundt int register_trapped_io(struct trapped_io *tiop) 42e7cc9a73SMagnus Damm { 43e7cc9a73SMagnus Damm struct resource *res; 44e7cc9a73SMagnus Damm unsigned long len = 0, flags = 0; 45e7cc9a73SMagnus Damm struct page *pages[TRAPPED_PAGES_MAX]; 46e7cc9a73SMagnus Damm int k, n; 47e7cc9a73SMagnus Damm 48eeee7853SPaul Mundt if (unlikely(trapped_io_disable)) 49eeee7853SPaul Mundt return 0; 50eeee7853SPaul Mundt 51e7cc9a73SMagnus Damm /* structure must be page aligned */ 52e7cc9a73SMagnus Damm if ((unsigned long)tiop & (PAGE_SIZE - 1)) 53e7cc9a73SMagnus Damm goto bad; 54e7cc9a73SMagnus Damm 55e7cc9a73SMagnus Damm for (k = 0; k < tiop->num_resources; k++) { 56e7cc9a73SMagnus Damm res = tiop->resource + k; 5728f65c11SJoe Perches len += roundup(resource_size(res), PAGE_SIZE); 58e7cc9a73SMagnus Damm flags |= res->flags; 59e7cc9a73SMagnus Damm } 60e7cc9a73SMagnus Damm 61e7cc9a73SMagnus Damm /* support IORESOURCE_IO _or_ MEM, not both */ 62e7cc9a73SMagnus Damm if (hweight_long(flags) != 1) 63e7cc9a73SMagnus Damm goto bad; 64e7cc9a73SMagnus Damm 65e7cc9a73SMagnus Damm n = len >> PAGE_SHIFT; 66e7cc9a73SMagnus Damm 67e7cc9a73SMagnus Damm if (n >= TRAPPED_PAGES_MAX) 68e7cc9a73SMagnus Damm goto bad; 69e7cc9a73SMagnus Damm 70e7cc9a73SMagnus Damm for (k = 0; k < n; k++) 71e7cc9a73SMagnus Damm pages[k] = virt_to_page(tiop); 72e7cc9a73SMagnus Damm 73e7cc9a73SMagnus Damm tiop->virt_base = vmap(pages, n, VM_MAP, PAGE_NONE); 74e7cc9a73SMagnus Damm if (!tiop->virt_base) 75e7cc9a73SMagnus Damm goto bad; 76e7cc9a73SMagnus Damm 77e7cc9a73SMagnus Damm len = 0; 78e7cc9a73SMagnus Damm for (k = 0; k < tiop->num_resources; k++) { 79e7cc9a73SMagnus Damm res = tiop->resource + k; 80e7cc9a73SMagnus Damm pr_info("trapped io 0x%08lx overrides %s 0x%08lx\n", 81e7cc9a73SMagnus Damm (unsigned long)(tiop->virt_base + len), 82e7cc9a73SMagnus Damm res->flags & IORESOURCE_IO ? "io" : "mmio", 83e7cc9a73SMagnus Damm (unsigned long)res->start); 8428f65c11SJoe Perches len += roundup(resource_size(res), PAGE_SIZE); 85e7cc9a73SMagnus Damm } 86e7cc9a73SMagnus Damm 87e7cc9a73SMagnus Damm tiop->magic = IO_TRAPPED_MAGIC; 88e7cc9a73SMagnus Damm INIT_LIST_HEAD(&tiop->list); 89e7cc9a73SMagnus Damm spin_lock_irq(&trapped_lock); 90ce816fa8SUwe Kleine-König #ifdef CONFIG_HAS_IOPORT_MAP 91e7cc9a73SMagnus Damm if (flags & IORESOURCE_IO) 92e7cc9a73SMagnus Damm list_add(&tiop->list, &trapped_io); 9386e4dd5aSPaul Mundt #endif 9486e4dd5aSPaul Mundt #ifdef CONFIG_HAS_IOMEM 95e7cc9a73SMagnus Damm if (flags & IORESOURCE_MEM) 96e7cc9a73SMagnus Damm list_add(&tiop->list, &trapped_mem); 9786e4dd5aSPaul Mundt #endif 98e7cc9a73SMagnus Damm spin_unlock_irq(&trapped_lock); 99e7cc9a73SMagnus Damm 100e7cc9a73SMagnus Damm return 0; 101e7cc9a73SMagnus Damm bad: 102*6d80f20cSKefeng Wang pr_warn("unable to install trapped io filter\n"); 103e7cc9a73SMagnus Damm return -1; 104e7cc9a73SMagnus Damm } 105ecc14e8cSPaul Mundt EXPORT_SYMBOL_GPL(register_trapped_io); 106e7cc9a73SMagnus Damm 107e7cc9a73SMagnus Damm void __iomem *match_trapped_io_handler(struct list_head *list, 108e7cc9a73SMagnus Damm unsigned long offset, 109e7cc9a73SMagnus Damm unsigned long size) 110e7cc9a73SMagnus Damm { 111e7cc9a73SMagnus Damm unsigned long voffs; 112e7cc9a73SMagnus Damm struct trapped_io *tiop; 113e7cc9a73SMagnus Damm struct resource *res; 114e7cc9a73SMagnus Damm int k, len; 115fd78a76aSStuart Menefy unsigned long flags; 116e7cc9a73SMagnus Damm 117fd78a76aSStuart Menefy spin_lock_irqsave(&trapped_lock, flags); 118e7cc9a73SMagnus Damm list_for_each_entry(tiop, list, list) { 119e7cc9a73SMagnus Damm voffs = 0; 120e7cc9a73SMagnus Damm for (k = 0; k < tiop->num_resources; k++) { 121e7cc9a73SMagnus Damm res = tiop->resource + k; 122e7cc9a73SMagnus Damm if (res->start == offset) { 123fd78a76aSStuart Menefy spin_unlock_irqrestore(&trapped_lock, flags); 124e7cc9a73SMagnus Damm return tiop->virt_base + voffs; 125e7cc9a73SMagnus Damm } 126e7cc9a73SMagnus Damm 12728f65c11SJoe Perches len = resource_size(res); 128e7cc9a73SMagnus Damm voffs += roundup(len, PAGE_SIZE); 129e7cc9a73SMagnus Damm } 130e7cc9a73SMagnus Damm } 131fd78a76aSStuart Menefy spin_unlock_irqrestore(&trapped_lock, flags); 132e7cc9a73SMagnus Damm return NULL; 133e7cc9a73SMagnus Damm } 134ecc14e8cSPaul Mundt EXPORT_SYMBOL_GPL(match_trapped_io_handler); 135e7cc9a73SMagnus Damm 136e7cc9a73SMagnus Damm static struct trapped_io *lookup_tiop(unsigned long address) 137e7cc9a73SMagnus Damm { 138e7cc9a73SMagnus Damm pgd_t *pgd_k; 139e7cc9a73SMagnus Damm pud_t *pud_k; 140e7cc9a73SMagnus Damm pmd_t *pmd_k; 141e7cc9a73SMagnus Damm pte_t *pte_k; 142e7cc9a73SMagnus Damm pte_t entry; 143e7cc9a73SMagnus Damm 144e7cc9a73SMagnus Damm pgd_k = swapper_pg_dir + pgd_index(address); 145e7cc9a73SMagnus Damm if (!pgd_present(*pgd_k)) 146e7cc9a73SMagnus Damm return NULL; 147e7cc9a73SMagnus Damm 148e7cc9a73SMagnus Damm pud_k = pud_offset(pgd_k, address); 149e7cc9a73SMagnus Damm if (!pud_present(*pud_k)) 150e7cc9a73SMagnus Damm return NULL; 151e7cc9a73SMagnus Damm 152e7cc9a73SMagnus Damm pmd_k = pmd_offset(pud_k, address); 153e7cc9a73SMagnus Damm if (!pmd_present(*pmd_k)) 154e7cc9a73SMagnus Damm return NULL; 155e7cc9a73SMagnus Damm 156e7cc9a73SMagnus Damm pte_k = pte_offset_kernel(pmd_k, address); 157e7cc9a73SMagnus Damm entry = *pte_k; 158e7cc9a73SMagnus Damm 159e7cc9a73SMagnus Damm return pfn_to_kaddr(pte_pfn(entry)); 160e7cc9a73SMagnus Damm } 161e7cc9a73SMagnus Damm 162e7cc9a73SMagnus Damm static unsigned long lookup_address(struct trapped_io *tiop, 163e7cc9a73SMagnus Damm unsigned long address) 164e7cc9a73SMagnus Damm { 165e7cc9a73SMagnus Damm struct resource *res; 166e7cc9a73SMagnus Damm unsigned long vaddr = (unsigned long)tiop->virt_base; 167e7cc9a73SMagnus Damm unsigned long len; 168e7cc9a73SMagnus Damm int k; 169e7cc9a73SMagnus Damm 170e7cc9a73SMagnus Damm for (k = 0; k < tiop->num_resources; k++) { 171e7cc9a73SMagnus Damm res = tiop->resource + k; 17228f65c11SJoe Perches len = roundup(resource_size(res), PAGE_SIZE); 173e7cc9a73SMagnus Damm if (address < (vaddr + len)) 174e7cc9a73SMagnus Damm return res->start + (address - vaddr); 175e7cc9a73SMagnus Damm vaddr += len; 176e7cc9a73SMagnus Damm } 177e7cc9a73SMagnus Damm return 0; 178e7cc9a73SMagnus Damm } 179e7cc9a73SMagnus Damm 180e7cc9a73SMagnus Damm static unsigned long long copy_word(unsigned long src_addr, int src_len, 181e7cc9a73SMagnus Damm unsigned long dst_addr, int dst_len) 182e7cc9a73SMagnus Damm { 183e7cc9a73SMagnus Damm unsigned long long tmp = 0; 184e7cc9a73SMagnus Damm 185e7cc9a73SMagnus Damm switch (src_len) { 186e7cc9a73SMagnus Damm case 1: 1879d56dd3bSPaul Mundt tmp = __raw_readb(src_addr); 188e7cc9a73SMagnus Damm break; 189e7cc9a73SMagnus Damm case 2: 1909d56dd3bSPaul Mundt tmp = __raw_readw(src_addr); 191e7cc9a73SMagnus Damm break; 192e7cc9a73SMagnus Damm case 4: 1939d56dd3bSPaul Mundt tmp = __raw_readl(src_addr); 194e7cc9a73SMagnus Damm break; 195e7cc9a73SMagnus Damm case 8: 1969d56dd3bSPaul Mundt tmp = __raw_readq(src_addr); 197e7cc9a73SMagnus Damm break; 198e7cc9a73SMagnus Damm } 199e7cc9a73SMagnus Damm 200e7cc9a73SMagnus Damm switch (dst_len) { 201e7cc9a73SMagnus Damm case 1: 2029d56dd3bSPaul Mundt __raw_writeb(tmp, dst_addr); 203e7cc9a73SMagnus Damm break; 204e7cc9a73SMagnus Damm case 2: 2059d56dd3bSPaul Mundt __raw_writew(tmp, dst_addr); 206e7cc9a73SMagnus Damm break; 207e7cc9a73SMagnus Damm case 4: 2089d56dd3bSPaul Mundt __raw_writel(tmp, dst_addr); 209e7cc9a73SMagnus Damm break; 210e7cc9a73SMagnus Damm case 8: 2119d56dd3bSPaul Mundt __raw_writeq(tmp, dst_addr); 212e7cc9a73SMagnus Damm break; 213e7cc9a73SMagnus Damm } 214e7cc9a73SMagnus Damm 215e7cc9a73SMagnus Damm return tmp; 216e7cc9a73SMagnus Damm } 217e7cc9a73SMagnus Damm 218e7cc9a73SMagnus Damm static unsigned long from_device(void *dst, const void *src, unsigned long cnt) 219e7cc9a73SMagnus Damm { 220e7cc9a73SMagnus Damm struct trapped_io *tiop; 221e7cc9a73SMagnus Damm unsigned long src_addr = (unsigned long)src; 222e7cc9a73SMagnus Damm unsigned long long tmp; 223e7cc9a73SMagnus Damm 224e7cc9a73SMagnus Damm pr_debug("trapped io read 0x%08lx (%ld)\n", src_addr, cnt); 225e7cc9a73SMagnus Damm tiop = lookup_tiop(src_addr); 226e7cc9a73SMagnus Damm WARN_ON(!tiop || (tiop->magic != IO_TRAPPED_MAGIC)); 227e7cc9a73SMagnus Damm 228e7cc9a73SMagnus Damm src_addr = lookup_address(tiop, src_addr); 229e7cc9a73SMagnus Damm if (!src_addr) 230e7cc9a73SMagnus Damm return cnt; 231e7cc9a73SMagnus Damm 232f1cdd63fSPaul Mundt tmp = copy_word(src_addr, 233f1cdd63fSPaul Mundt max_t(unsigned long, cnt, 234f1cdd63fSPaul Mundt (tiop->minimum_bus_width / 8)), 235e7cc9a73SMagnus Damm (unsigned long)dst, cnt); 236e7cc9a73SMagnus Damm 237e7cc9a73SMagnus Damm pr_debug("trapped io read 0x%08lx -> 0x%08llx\n", src_addr, tmp); 238e7cc9a73SMagnus Damm return 0; 239e7cc9a73SMagnus Damm } 240e7cc9a73SMagnus Damm 241e7cc9a73SMagnus Damm static unsigned long to_device(void *dst, const void *src, unsigned long cnt) 242e7cc9a73SMagnus Damm { 243e7cc9a73SMagnus Damm struct trapped_io *tiop; 244e7cc9a73SMagnus Damm unsigned long dst_addr = (unsigned long)dst; 245e7cc9a73SMagnus Damm unsigned long long tmp; 246e7cc9a73SMagnus Damm 247e7cc9a73SMagnus Damm pr_debug("trapped io write 0x%08lx (%ld)\n", dst_addr, cnt); 248e7cc9a73SMagnus Damm tiop = lookup_tiop(dst_addr); 249e7cc9a73SMagnus Damm WARN_ON(!tiop || (tiop->magic != IO_TRAPPED_MAGIC)); 250e7cc9a73SMagnus Damm 251e7cc9a73SMagnus Damm dst_addr = lookup_address(tiop, dst_addr); 252e7cc9a73SMagnus Damm if (!dst_addr) 253e7cc9a73SMagnus Damm return cnt; 254e7cc9a73SMagnus Damm 255e7cc9a73SMagnus Damm tmp = copy_word((unsigned long)src, cnt, 256f1cdd63fSPaul Mundt dst_addr, max_t(unsigned long, cnt, 257f1cdd63fSPaul Mundt (tiop->minimum_bus_width / 8))); 258e7cc9a73SMagnus Damm 259e7cc9a73SMagnus Damm pr_debug("trapped io write 0x%08lx -> 0x%08llx\n", dst_addr, tmp); 260e7cc9a73SMagnus Damm return 0; 261e7cc9a73SMagnus Damm } 262e7cc9a73SMagnus Damm 263e7cc9a73SMagnus Damm static struct mem_access trapped_io_access = { 264e7cc9a73SMagnus Damm from_device, 265e7cc9a73SMagnus Damm to_device, 266e7cc9a73SMagnus Damm }; 267e7cc9a73SMagnus Damm 268e7cc9a73SMagnus Damm int handle_trapped_io(struct pt_regs *regs, unsigned long address) 269e7cc9a73SMagnus Damm { 270e7cc9a73SMagnus Damm mm_segment_t oldfs; 2712bcfffa4SPaul Mundt insn_size_t instruction; 272e7cc9a73SMagnus Damm int tmp; 273e7cc9a73SMagnus Damm 27408b36c4aSPaul Mundt if (trapped_io_disable) 27508b36c4aSPaul Mundt return 0; 276e7cc9a73SMagnus Damm if (!lookup_tiop(address)) 277e7cc9a73SMagnus Damm return 0; 278e7cc9a73SMagnus Damm 279e7cc9a73SMagnus Damm WARN_ON(user_mode(regs)); 280e7cc9a73SMagnus Damm 281e7cc9a73SMagnus Damm oldfs = get_fs(); 282e7cc9a73SMagnus Damm set_fs(KERNEL_DS); 283e7cc9a73SMagnus Damm if (copy_from_user(&instruction, (void *)(regs->pc), 284e7cc9a73SMagnus Damm sizeof(instruction))) { 285e7cc9a73SMagnus Damm set_fs(oldfs); 286e7cc9a73SMagnus Damm return 0; 287e7cc9a73SMagnus Damm } 288e7cc9a73SMagnus Damm 2894aa5ac4eSMatt Fleming tmp = handle_unaligned_access(instruction, regs, 290ace2dc7dSPaul Mundt &trapped_io_access, 1, address); 291e7cc9a73SMagnus Damm set_fs(oldfs); 292e7cc9a73SMagnus Damm return tmp == 0; 293e7cc9a73SMagnus Damm } 294