1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implement the default iomap interfaces 4 * 5 * (C) Copyright 2004 Linus Torvalds 6 */ 7 #include <linux/pci.h> 8 #include <linux/io.h> 9 #include <linux/kmsan-checks.h> 10 11 #include <linux/export.h> 12 13 /* 14 * Read/write from/to an (offsettable) iomem cookie. It might be a PIO 15 * access or a MMIO access, these functions don't care. The info is 16 * encoded in the hardware mapping set up by the mapping functions 17 * (or the cookie itself, depending on implementation and hw). 18 * 19 * The generic routines don't assume any hardware mappings, and just 20 * encode the PIO/MMIO as part of the cookie. They coldly assume that 21 * the MMIO IO mappings are not in the low address range. 22 * 23 * Architectures for which this is not true can't use this generic 24 * implementation and should do their own copy. 25 */ 26 27 #ifndef HAVE_ARCH_PIO_SIZE 28 /* 29 * We encode the physical PIO addresses (0-0xffff) into the 30 * pointer by offsetting them with a constant (0x10000) and 31 * assuming that all the low addresses are always PIO. That means 32 * we can do some sanity checks on the low bits, and don't 33 * need to just take things for granted. 34 */ 35 #define PIO_OFFSET 0x10000UL 36 #define PIO_MASK 0x0ffffUL 37 #define PIO_RESERVED 0x40000UL 38 #endif 39 40 static void bad_io_access(unsigned long port, const char *access) 41 { 42 static int count = 10; 43 if (count) { 44 count--; 45 WARN(1, KERN_ERR "Bad IO access at port %#lx (%s)\n", port, access); 46 } 47 } 48 49 /* 50 * Ugly macros are a way of life. 51 */ 52 #define IO_COND(addr, is_pio, is_mmio) do { \ 53 unsigned long port = (unsigned long __force)addr; \ 54 if (port >= PIO_RESERVED) { \ 55 is_mmio; \ 56 } else if (port > PIO_OFFSET) { \ 57 port &= PIO_MASK; \ 58 is_pio; \ 59 } else \ 60 bad_io_access(port, #is_pio ); \ 61 } while (0) 62 63 #ifndef pio_read16be 64 #define pio_read16be(port) swab16(inw(port)) 65 #define pio_read32be(port) swab32(inl(port)) 66 #endif 67 68 #ifndef mmio_read16be 69 #define mmio_read16be(addr) swab16(readw(addr)) 70 #define mmio_read32be(addr) swab32(readl(addr)) 71 #define mmio_read64be(addr) swab64(readq(addr)) 72 #endif 73 74 /* 75 * Here and below, we apply __no_kmsan_checks to functions reading data from 76 * hardware, to ensure that KMSAN marks their return values as initialized. 77 */ 78 __no_kmsan_checks 79 unsigned int ioread8(const void __iomem *addr) 80 { 81 IO_COND(addr, return inb(port), return readb(addr)); 82 return 0xff; 83 } 84 __no_kmsan_checks 85 unsigned int ioread16(const void __iomem *addr) 86 { 87 IO_COND(addr, return inw(port), return readw(addr)); 88 return 0xffff; 89 } 90 __no_kmsan_checks 91 unsigned int ioread16be(const void __iomem *addr) 92 { 93 IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr)); 94 return 0xffff; 95 } 96 __no_kmsan_checks 97 unsigned int ioread32(const void __iomem *addr) 98 { 99 IO_COND(addr, return inl(port), return readl(addr)); 100 return 0xffffffff; 101 } 102 __no_kmsan_checks 103 unsigned int ioread32be(const void __iomem *addr) 104 { 105 IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr)); 106 return 0xffffffff; 107 } 108 EXPORT_SYMBOL(ioread8); 109 EXPORT_SYMBOL(ioread16); 110 EXPORT_SYMBOL(ioread16be); 111 EXPORT_SYMBOL(ioread32); 112 EXPORT_SYMBOL(ioread32be); 113 114 #ifdef readq 115 static u64 pio_read64_lo_hi(unsigned long port) 116 { 117 u64 lo, hi; 118 119 lo = inl(port); 120 hi = inl(port + sizeof(u32)); 121 122 return lo | (hi << 32); 123 } 124 125 static u64 pio_read64_hi_lo(unsigned long port) 126 { 127 u64 lo, hi; 128 129 hi = inl(port + sizeof(u32)); 130 lo = inl(port); 131 132 return lo | (hi << 32); 133 } 134 135 static u64 pio_read64be_lo_hi(unsigned long port) 136 { 137 u64 lo, hi; 138 139 lo = pio_read32be(port + sizeof(u32)); 140 hi = pio_read32be(port); 141 142 return lo | (hi << 32); 143 } 144 145 static u64 pio_read64be_hi_lo(unsigned long port) 146 { 147 u64 lo, hi; 148 149 hi = pio_read32be(port); 150 lo = pio_read32be(port + sizeof(u32)); 151 152 return lo | (hi << 32); 153 } 154 155 __no_kmsan_checks 156 u64 ioread64_lo_hi(const void __iomem *addr) 157 { 158 IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr)); 159 return 0xffffffffffffffffULL; 160 } 161 162 __no_kmsan_checks 163 u64 ioread64_hi_lo(const void __iomem *addr) 164 { 165 IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr)); 166 return 0xffffffffffffffffULL; 167 } 168 169 __no_kmsan_checks 170 u64 ioread64be_lo_hi(const void __iomem *addr) 171 { 172 IO_COND(addr, return pio_read64be_lo_hi(port), 173 return mmio_read64be(addr)); 174 return 0xffffffffffffffffULL; 175 } 176 177 __no_kmsan_checks 178 u64 ioread64be_hi_lo(const void __iomem *addr) 179 { 180 IO_COND(addr, return pio_read64be_hi_lo(port), 181 return mmio_read64be(addr)); 182 return 0xffffffffffffffffULL; 183 } 184 185 EXPORT_SYMBOL(ioread64_lo_hi); 186 EXPORT_SYMBOL(ioread64_hi_lo); 187 EXPORT_SYMBOL(ioread64be_lo_hi); 188 EXPORT_SYMBOL(ioread64be_hi_lo); 189 190 #endif /* readq */ 191 192 #ifndef pio_write16be 193 #define pio_write16be(val,port) outw(swab16(val),port) 194 #define pio_write32be(val,port) outl(swab32(val),port) 195 #endif 196 197 #ifndef mmio_write16be 198 #define mmio_write16be(val,port) writew(swab16(val),port) 199 #define mmio_write32be(val,port) writel(swab32(val),port) 200 #define mmio_write64be(val,port) writeq(swab64(val),port) 201 #endif 202 203 void iowrite8(u8 val, void __iomem *addr) 204 { 205 /* Make sure uninitialized memory isn't copied to devices. */ 206 kmsan_check_memory(&val, sizeof(val)); 207 IO_COND(addr, outb(val,port), writeb(val, addr)); 208 } 209 void iowrite16(u16 val, void __iomem *addr) 210 { 211 /* Make sure uninitialized memory isn't copied to devices. */ 212 kmsan_check_memory(&val, sizeof(val)); 213 IO_COND(addr, outw(val,port), writew(val, addr)); 214 } 215 void iowrite16be(u16 val, void __iomem *addr) 216 { 217 /* Make sure uninitialized memory isn't copied to devices. */ 218 kmsan_check_memory(&val, sizeof(val)); 219 IO_COND(addr, pio_write16be(val,port), mmio_write16be(val, addr)); 220 } 221 void iowrite32(u32 val, void __iomem *addr) 222 { 223 /* Make sure uninitialized memory isn't copied to devices. */ 224 kmsan_check_memory(&val, sizeof(val)); 225 IO_COND(addr, outl(val,port), writel(val, addr)); 226 } 227 void iowrite32be(u32 val, void __iomem *addr) 228 { 229 /* Make sure uninitialized memory isn't copied to devices. */ 230 kmsan_check_memory(&val, sizeof(val)); 231 IO_COND(addr, pio_write32be(val,port), mmio_write32be(val, addr)); 232 } 233 EXPORT_SYMBOL(iowrite8); 234 EXPORT_SYMBOL(iowrite16); 235 EXPORT_SYMBOL(iowrite16be); 236 EXPORT_SYMBOL(iowrite32); 237 EXPORT_SYMBOL(iowrite32be); 238 239 #ifdef writeq 240 static void pio_write64_lo_hi(u64 val, unsigned long port) 241 { 242 outl(val, port); 243 outl(val >> 32, port + sizeof(u32)); 244 } 245 246 static void pio_write64_hi_lo(u64 val, unsigned long port) 247 { 248 outl(val >> 32, port + sizeof(u32)); 249 outl(val, port); 250 } 251 252 static void pio_write64be_lo_hi(u64 val, unsigned long port) 253 { 254 pio_write32be(val, port + sizeof(u32)); 255 pio_write32be(val >> 32, port); 256 } 257 258 static void pio_write64be_hi_lo(u64 val, unsigned long port) 259 { 260 pio_write32be(val >> 32, port); 261 pio_write32be(val, port + sizeof(u32)); 262 } 263 264 void iowrite64_lo_hi(u64 val, void __iomem *addr) 265 { 266 /* Make sure uninitialized memory isn't copied to devices. */ 267 kmsan_check_memory(&val, sizeof(val)); 268 IO_COND(addr, pio_write64_lo_hi(val, port), 269 writeq(val, addr)); 270 } 271 272 void iowrite64_hi_lo(u64 val, void __iomem *addr) 273 { 274 /* Make sure uninitialized memory isn't copied to devices. */ 275 kmsan_check_memory(&val, sizeof(val)); 276 IO_COND(addr, pio_write64_hi_lo(val, port), 277 writeq(val, addr)); 278 } 279 280 void iowrite64be_lo_hi(u64 val, void __iomem *addr) 281 { 282 /* Make sure uninitialized memory isn't copied to devices. */ 283 kmsan_check_memory(&val, sizeof(val)); 284 IO_COND(addr, pio_write64be_lo_hi(val, port), 285 mmio_write64be(val, addr)); 286 } 287 288 void iowrite64be_hi_lo(u64 val, void __iomem *addr) 289 { 290 /* Make sure uninitialized memory isn't copied to devices. */ 291 kmsan_check_memory(&val, sizeof(val)); 292 IO_COND(addr, pio_write64be_hi_lo(val, port), 293 mmio_write64be(val, addr)); 294 } 295 296 EXPORT_SYMBOL(iowrite64_lo_hi); 297 EXPORT_SYMBOL(iowrite64_hi_lo); 298 EXPORT_SYMBOL(iowrite64be_lo_hi); 299 EXPORT_SYMBOL(iowrite64be_hi_lo); 300 301 #endif /* readq */ 302 303 /* 304 * These are the "repeat MMIO read/write" functions. 305 * Note the "__raw" accesses, since we don't want to 306 * convert to CPU byte order. We write in "IO byte 307 * order" (we also don't have IO barriers). 308 */ 309 #ifndef mmio_insb 310 static inline void mmio_insb(const void __iomem *addr, u8 *dst, int count) 311 { 312 while (--count >= 0) { 313 u8 data = __raw_readb(addr); 314 *dst = data; 315 dst++; 316 } 317 } 318 static inline void mmio_insw(const void __iomem *addr, u16 *dst, int count) 319 { 320 while (--count >= 0) { 321 u16 data = __raw_readw(addr); 322 *dst = data; 323 dst++; 324 } 325 } 326 static inline void mmio_insl(const void __iomem *addr, u32 *dst, int count) 327 { 328 while (--count >= 0) { 329 u32 data = __raw_readl(addr); 330 *dst = data; 331 dst++; 332 } 333 } 334 #endif 335 336 #ifndef mmio_outsb 337 static inline void mmio_outsb(void __iomem *addr, const u8 *src, int count) 338 { 339 while (--count >= 0) { 340 __raw_writeb(*src, addr); 341 src++; 342 } 343 } 344 static inline void mmio_outsw(void __iomem *addr, const u16 *src, int count) 345 { 346 while (--count >= 0) { 347 __raw_writew(*src, addr); 348 src++; 349 } 350 } 351 static inline void mmio_outsl(void __iomem *addr, const u32 *src, int count) 352 { 353 while (--count >= 0) { 354 __raw_writel(*src, addr); 355 src++; 356 } 357 } 358 #endif 359 360 void ioread8_rep(const void __iomem *addr, void *dst, unsigned long count) 361 { 362 IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count)); 363 /* KMSAN must treat values read from devices as initialized. */ 364 kmsan_unpoison_memory(dst, count); 365 } 366 void ioread16_rep(const void __iomem *addr, void *dst, unsigned long count) 367 { 368 IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count)); 369 /* KMSAN must treat values read from devices as initialized. */ 370 kmsan_unpoison_memory(dst, count * 2); 371 } 372 void ioread32_rep(const void __iomem *addr, void *dst, unsigned long count) 373 { 374 IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count)); 375 /* KMSAN must treat values read from devices as initialized. */ 376 kmsan_unpoison_memory(dst, count * 4); 377 } 378 EXPORT_SYMBOL(ioread8_rep); 379 EXPORT_SYMBOL(ioread16_rep); 380 EXPORT_SYMBOL(ioread32_rep); 381 382 void iowrite8_rep(void __iomem *addr, const void *src, unsigned long count) 383 { 384 /* Make sure uninitialized memory isn't copied to devices. */ 385 kmsan_check_memory(src, count); 386 IO_COND(addr, outsb(port, src, count), mmio_outsb(addr, src, count)); 387 } 388 void iowrite16_rep(void __iomem *addr, const void *src, unsigned long count) 389 { 390 /* Make sure uninitialized memory isn't copied to devices. */ 391 kmsan_check_memory(src, count * 2); 392 IO_COND(addr, outsw(port, src, count), mmio_outsw(addr, src, count)); 393 } 394 void iowrite32_rep(void __iomem *addr, const void *src, unsigned long count) 395 { 396 /* Make sure uninitialized memory isn't copied to devices. */ 397 kmsan_check_memory(src, count * 4); 398 IO_COND(addr, outsl(port, src,count), mmio_outsl(addr, src, count)); 399 } 400 EXPORT_SYMBOL(iowrite8_rep); 401 EXPORT_SYMBOL(iowrite16_rep); 402 EXPORT_SYMBOL(iowrite32_rep); 403 404 #ifdef CONFIG_HAS_IOPORT_MAP 405 /* Create a virtual mapping cookie for an IO port range */ 406 void __iomem *ioport_map(unsigned long port, unsigned int nr) 407 { 408 if (port > PIO_MASK) 409 return NULL; 410 return (void __iomem *) (unsigned long) (port + PIO_OFFSET); 411 } 412 413 void ioport_unmap(void __iomem *addr) 414 { 415 /* Nothing to do */ 416 } 417 EXPORT_SYMBOL(ioport_map); 418 EXPORT_SYMBOL(ioport_unmap); 419 #endif /* CONFIG_HAS_IOPORT_MAP */ 420 421 #ifdef CONFIG_PCI 422 /* Hide the details if this is a MMIO or PIO address space and just do what 423 * you expect in the correct way. */ 424 void pci_iounmap(struct pci_dev *dev, void __iomem * addr) 425 { 426 IO_COND(addr, /* nothing */, iounmap(addr)); 427 } 428 EXPORT_SYMBOL(pci_iounmap); 429 #endif /* CONFIG_PCI */ 430