1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * PS3 address space management. 4 * 5 * Copyright (C) 2006 Sony Computer Entertainment Inc. 6 * Copyright 2006 Sony Corp. 7 */ 8 9 #include <linux/dma-mapping.h> 10 #include <linux/kernel.h> 11 #include <linux/export.h> 12 #include <linux/memblock.h> 13 #include <linux/slab.h> 14 15 #include <asm/cell-regs.h> 16 #include <asm/firmware.h> 17 #include <asm/prom.h> 18 #include <asm/udbg.h> 19 #include <asm/lv1call.h> 20 #include <asm/setup.h> 21 22 #include "platform.h" 23 24 #if defined(DEBUG) 25 #define DBG udbg_printf 26 #else 27 #define DBG pr_devel 28 #endif 29 30 enum { 31 #if defined(CONFIG_PS3_DYNAMIC_DMA) 32 USE_DYNAMIC_DMA = 1, 33 #else 34 USE_DYNAMIC_DMA = 0, 35 #endif 36 }; 37 38 enum { 39 PAGE_SHIFT_4K = 12U, 40 PAGE_SHIFT_64K = 16U, 41 PAGE_SHIFT_16M = 24U, 42 }; 43 44 static unsigned long make_page_sizes(unsigned long a, unsigned long b) 45 { 46 return (a << 56) | (b << 48); 47 } 48 49 enum { 50 ALLOCATE_MEMORY_TRY_ALT_UNIT = 0X04, 51 ALLOCATE_MEMORY_ADDR_ZERO = 0X08, 52 }; 53 54 /* valid htab sizes are {18,19,20} = 256K, 512K, 1M */ 55 56 enum { 57 HTAB_SIZE_MAX = 20U, /* HV limit of 1MB */ 58 HTAB_SIZE_MIN = 18U, /* CPU limit of 256KB */ 59 }; 60 61 /*============================================================================*/ 62 /* virtual address space routines */ 63 /*============================================================================*/ 64 65 /** 66 * struct mem_region - memory region structure 67 * @base: base address 68 * @size: size in bytes 69 * @offset: difference between base and rm.size 70 * @destroy: flag if region should be destroyed upon shutdown 71 */ 72 73 struct mem_region { 74 u64 base; 75 u64 size; 76 unsigned long offset; 77 int destroy; 78 }; 79 80 /** 81 * struct map - address space state variables holder 82 * @total: total memory available as reported by HV 83 * @vas_id - HV virtual address space id 84 * @htab_size: htab size in bytes 85 * 86 * The HV virtual address space (vas) allows for hotplug memory regions. 87 * Memory regions can be created and destroyed in the vas at runtime. 88 * @rm: real mode (bootmem) region 89 * @r1: highmem region(s) 90 * 91 * ps3 addresses 92 * virt_addr: a cpu 'translated' effective address 93 * phys_addr: an address in what Linux thinks is the physical address space 94 * lpar_addr: an address in the HV virtual address space 95 * bus_addr: an io controller 'translated' address on a device bus 96 */ 97 98 struct map { 99 u64 total; 100 u64 vas_id; 101 u64 htab_size; 102 struct mem_region rm; 103 struct mem_region r1; 104 }; 105 106 #define debug_dump_map(x) _debug_dump_map(x, __func__, __LINE__) 107 static void __maybe_unused _debug_dump_map(const struct map *m, 108 const char *func, int line) 109 { 110 DBG("%s:%d: map.total = %llxh\n", func, line, m->total); 111 DBG("%s:%d: map.rm.size = %llxh\n", func, line, m->rm.size); 112 DBG("%s:%d: map.vas_id = %llu\n", func, line, m->vas_id); 113 DBG("%s:%d: map.htab_size = %llxh\n", func, line, m->htab_size); 114 DBG("%s:%d: map.r1.base = %llxh\n", func, line, m->r1.base); 115 DBG("%s:%d: map.r1.offset = %lxh\n", func, line, m->r1.offset); 116 DBG("%s:%d: map.r1.size = %llxh\n", func, line, m->r1.size); 117 } 118 119 static struct map map; 120 121 /** 122 * ps3_mm_phys_to_lpar - translate a linux physical address to lpar address 123 * @phys_addr: linux physical address 124 */ 125 126 unsigned long ps3_mm_phys_to_lpar(unsigned long phys_addr) 127 { 128 BUG_ON(is_kernel_addr(phys_addr)); 129 return (phys_addr < map.rm.size || phys_addr >= map.total) 130 ? phys_addr : phys_addr + map.r1.offset; 131 } 132 133 EXPORT_SYMBOL(ps3_mm_phys_to_lpar); 134 135 /** 136 * ps3_mm_vas_create - create the virtual address space 137 */ 138 139 void __init ps3_mm_vas_create(unsigned long* htab_size) 140 { 141 int result; 142 u64 start_address; 143 u64 size; 144 u64 access_right; 145 u64 max_page_size; 146 u64 flags; 147 148 result = lv1_query_logical_partition_address_region_info(0, 149 &start_address, &size, &access_right, &max_page_size, 150 &flags); 151 152 if (result) { 153 DBG("%s:%d: lv1_query_logical_partition_address_region_info " 154 "failed: %s\n", __func__, __LINE__, 155 ps3_result(result)); 156 goto fail; 157 } 158 159 if (max_page_size < PAGE_SHIFT_16M) { 160 DBG("%s:%d: bad max_page_size %llxh\n", __func__, __LINE__, 161 max_page_size); 162 goto fail; 163 } 164 165 BUILD_BUG_ON(CONFIG_PS3_HTAB_SIZE > HTAB_SIZE_MAX); 166 BUILD_BUG_ON(CONFIG_PS3_HTAB_SIZE < HTAB_SIZE_MIN); 167 168 result = lv1_construct_virtual_address_space(CONFIG_PS3_HTAB_SIZE, 169 2, make_page_sizes(PAGE_SHIFT_16M, PAGE_SHIFT_64K), 170 &map.vas_id, &map.htab_size); 171 172 if (result) { 173 DBG("%s:%d: lv1_construct_virtual_address_space failed: %s\n", 174 __func__, __LINE__, ps3_result(result)); 175 goto fail; 176 } 177 178 result = lv1_select_virtual_address_space(map.vas_id); 179 180 if (result) { 181 DBG("%s:%d: lv1_select_virtual_address_space failed: %s\n", 182 __func__, __LINE__, ps3_result(result)); 183 goto fail; 184 } 185 186 *htab_size = map.htab_size; 187 188 debug_dump_map(&map); 189 190 return; 191 192 fail: 193 panic("ps3_mm_vas_create failed"); 194 } 195 196 /** 197 * ps3_mm_vas_destroy - 198 */ 199 200 void ps3_mm_vas_destroy(void) 201 { 202 int result; 203 204 if (map.vas_id) { 205 result = lv1_select_virtual_address_space(0); 206 result += lv1_destruct_virtual_address_space(map.vas_id); 207 208 if (result) { 209 lv1_panic(0); 210 } 211 212 map.vas_id = 0; 213 } 214 } 215 216 static int ps3_mm_get_repository_highmem(struct mem_region *r) 217 { 218 int result; 219 220 /* Assume a single highmem region. */ 221 222 result = ps3_repository_read_highmem_info(0, &r->base, &r->size); 223 224 if (result) 225 goto zero_region; 226 227 if (!r->base || !r->size) { 228 result = -1; 229 goto zero_region; 230 } 231 232 r->offset = r->base - map.rm.size; 233 234 DBG("%s:%d: Found high region in repository: %llxh %llxh\n", 235 __func__, __LINE__, r->base, r->size); 236 237 return 0; 238 239 zero_region: 240 DBG("%s:%d: No high region in repository.\n", __func__, __LINE__); 241 242 r->size = r->base = r->offset = 0; 243 return result; 244 } 245 246 static int ps3_mm_set_repository_highmem(const struct mem_region *r) 247 { 248 /* Assume a single highmem region. */ 249 250 return r ? ps3_repository_write_highmem_info(0, r->base, r->size) : 251 ps3_repository_write_highmem_info(0, 0, 0); 252 } 253 254 /** 255 * ps3_mm_region_create - create a memory region in the vas 256 * @r: pointer to a struct mem_region to accept initialized values 257 * @size: requested region size 258 * 259 * This implementation creates the region with the vas large page size. 260 * @size is rounded down to a multiple of the vas large page size. 261 */ 262 263 static int ps3_mm_region_create(struct mem_region *r, unsigned long size) 264 { 265 int result; 266 u64 muid; 267 268 r->size = ALIGN_DOWN(size, 1 << PAGE_SHIFT_16M); 269 270 DBG("%s:%d requested %lxh\n", __func__, __LINE__, size); 271 DBG("%s:%d actual %llxh\n", __func__, __LINE__, r->size); 272 DBG("%s:%d difference %llxh (%lluMB)\n", __func__, __LINE__, 273 size - r->size, (size - r->size) / 1024 / 1024); 274 275 if (r->size == 0) { 276 DBG("%s:%d: size == 0\n", __func__, __LINE__); 277 result = -1; 278 goto zero_region; 279 } 280 281 result = lv1_allocate_memory(r->size, PAGE_SHIFT_16M, 0, 282 ALLOCATE_MEMORY_TRY_ALT_UNIT, &r->base, &muid); 283 284 if (result || r->base < map.rm.size) { 285 DBG("%s:%d: lv1_allocate_memory failed: %s\n", 286 __func__, __LINE__, ps3_result(result)); 287 goto zero_region; 288 } 289 290 r->destroy = 1; 291 r->offset = r->base - map.rm.size; 292 return result; 293 294 zero_region: 295 r->size = r->base = r->offset = 0; 296 return result; 297 } 298 299 /** 300 * ps3_mm_region_destroy - destroy a memory region 301 * @r: pointer to struct mem_region 302 */ 303 304 static void ps3_mm_region_destroy(struct mem_region *r) 305 { 306 int result; 307 308 if (!r->destroy) { 309 return; 310 } 311 312 if (r->base) { 313 result = lv1_release_memory(r->base); 314 315 if (result) { 316 lv1_panic(0); 317 } 318 319 r->size = r->base = r->offset = 0; 320 map.total = map.rm.size; 321 } 322 323 ps3_mm_set_repository_highmem(NULL); 324 } 325 326 /*============================================================================*/ 327 /* dma routines */ 328 /*============================================================================*/ 329 330 /** 331 * dma_sb_lpar_to_bus - Translate an lpar address to ioc mapped bus address. 332 * @r: pointer to dma region structure 333 * @lpar_addr: HV lpar address 334 */ 335 336 static unsigned long dma_sb_lpar_to_bus(struct ps3_dma_region *r, 337 unsigned long lpar_addr) 338 { 339 if (lpar_addr >= map.rm.size) 340 lpar_addr -= map.r1.offset; 341 BUG_ON(lpar_addr < r->offset); 342 BUG_ON(lpar_addr >= r->offset + r->len); 343 return r->bus_addr + lpar_addr - r->offset; 344 } 345 346 #define dma_dump_region(_a) _dma_dump_region(_a, __func__, __LINE__) 347 static void __maybe_unused _dma_dump_region(const struct ps3_dma_region *r, 348 const char *func, int line) 349 { 350 DBG("%s:%d: dev %llu:%llu\n", func, line, r->dev->bus_id, 351 r->dev->dev_id); 352 DBG("%s:%d: page_size %u\n", func, line, r->page_size); 353 DBG("%s:%d: bus_addr %lxh\n", func, line, r->bus_addr); 354 DBG("%s:%d: len %lxh\n", func, line, r->len); 355 DBG("%s:%d: offset %lxh\n", func, line, r->offset); 356 } 357 358 /** 359 * dma_chunk - A chunk of dma pages mapped by the io controller. 360 * @region - The dma region that owns this chunk. 361 * @lpar_addr: Starting lpar address of the area to map. 362 * @bus_addr: Starting ioc bus address of the area to map. 363 * @len: Length in bytes of the area to map. 364 * @link: A struct list_head used with struct ps3_dma_region.chunk_list, the 365 * list of all chuncks owned by the region. 366 * 367 * This implementation uses a very simple dma page manager 368 * based on the dma_chunk structure. This scheme assumes 369 * that all drivers use very well behaved dma ops. 370 */ 371 372 struct dma_chunk { 373 struct ps3_dma_region *region; 374 unsigned long lpar_addr; 375 unsigned long bus_addr; 376 unsigned long len; 377 struct list_head link; 378 unsigned int usage_count; 379 }; 380 381 #define dma_dump_chunk(_a) _dma_dump_chunk(_a, __func__, __LINE__) 382 static void _dma_dump_chunk (const struct dma_chunk* c, const char* func, 383 int line) 384 { 385 DBG("%s:%d: r.dev %llu:%llu\n", func, line, 386 c->region->dev->bus_id, c->region->dev->dev_id); 387 DBG("%s:%d: r.bus_addr %lxh\n", func, line, c->region->bus_addr); 388 DBG("%s:%d: r.page_size %u\n", func, line, c->region->page_size); 389 DBG("%s:%d: r.len %lxh\n", func, line, c->region->len); 390 DBG("%s:%d: r.offset %lxh\n", func, line, c->region->offset); 391 DBG("%s:%d: c.lpar_addr %lxh\n", func, line, c->lpar_addr); 392 DBG("%s:%d: c.bus_addr %lxh\n", func, line, c->bus_addr); 393 DBG("%s:%d: c.len %lxh\n", func, line, c->len); 394 } 395 396 static struct dma_chunk * dma_find_chunk(struct ps3_dma_region *r, 397 unsigned long bus_addr, unsigned long len) 398 { 399 struct dma_chunk *c; 400 unsigned long aligned_bus = ALIGN_DOWN(bus_addr, 1 << r->page_size); 401 unsigned long aligned_len = ALIGN(len+bus_addr-aligned_bus, 402 1 << r->page_size); 403 404 list_for_each_entry(c, &r->chunk_list.head, link) { 405 /* intersection */ 406 if (aligned_bus >= c->bus_addr && 407 aligned_bus + aligned_len <= c->bus_addr + c->len) 408 return c; 409 410 /* below */ 411 if (aligned_bus + aligned_len <= c->bus_addr) 412 continue; 413 414 /* above */ 415 if (aligned_bus >= c->bus_addr + c->len) 416 continue; 417 418 /* we don't handle the multi-chunk case for now */ 419 dma_dump_chunk(c); 420 BUG(); 421 } 422 return NULL; 423 } 424 425 static struct dma_chunk *dma_find_chunk_lpar(struct ps3_dma_region *r, 426 unsigned long lpar_addr, unsigned long len) 427 { 428 struct dma_chunk *c; 429 unsigned long aligned_lpar = ALIGN_DOWN(lpar_addr, 1 << r->page_size); 430 unsigned long aligned_len = ALIGN(len + lpar_addr - aligned_lpar, 431 1 << r->page_size); 432 433 list_for_each_entry(c, &r->chunk_list.head, link) { 434 /* intersection */ 435 if (c->lpar_addr <= aligned_lpar && 436 aligned_lpar < c->lpar_addr + c->len) { 437 if (aligned_lpar + aligned_len <= c->lpar_addr + c->len) 438 return c; 439 else { 440 dma_dump_chunk(c); 441 BUG(); 442 } 443 } 444 /* below */ 445 if (aligned_lpar + aligned_len <= c->lpar_addr) { 446 continue; 447 } 448 /* above */ 449 if (c->lpar_addr + c->len <= aligned_lpar) { 450 continue; 451 } 452 } 453 return NULL; 454 } 455 456 static int dma_sb_free_chunk(struct dma_chunk *c) 457 { 458 int result = 0; 459 460 if (c->bus_addr) { 461 result = lv1_unmap_device_dma_region(c->region->dev->bus_id, 462 c->region->dev->dev_id, c->bus_addr, c->len); 463 BUG_ON(result); 464 } 465 466 kfree(c); 467 return result; 468 } 469 470 static int dma_ioc0_free_chunk(struct dma_chunk *c) 471 { 472 int result = 0; 473 int iopage; 474 unsigned long offset; 475 struct ps3_dma_region *r = c->region; 476 477 DBG("%s:start\n", __func__); 478 for (iopage = 0; iopage < (c->len >> r->page_size); iopage++) { 479 offset = (1 << r->page_size) * iopage; 480 /* put INVALID entry */ 481 result = lv1_put_iopte(0, 482 c->bus_addr + offset, 483 c->lpar_addr + offset, 484 r->ioid, 485 0); 486 DBG("%s: bus=%#lx, lpar=%#lx, ioid=%d\n", __func__, 487 c->bus_addr + offset, 488 c->lpar_addr + offset, 489 r->ioid); 490 491 if (result) { 492 DBG("%s:%d: lv1_put_iopte failed: %s\n", __func__, 493 __LINE__, ps3_result(result)); 494 } 495 } 496 kfree(c); 497 DBG("%s:end\n", __func__); 498 return result; 499 } 500 501 /** 502 * dma_sb_map_pages - Maps dma pages into the io controller bus address space. 503 * @r: Pointer to a struct ps3_dma_region. 504 * @phys_addr: Starting physical address of the area to map. 505 * @len: Length in bytes of the area to map. 506 * c_out: A pointer to receive an allocated struct dma_chunk for this area. 507 * 508 * This is the lowest level dma mapping routine, and is the one that will 509 * make the HV call to add the pages into the io controller address space. 510 */ 511 512 static int dma_sb_map_pages(struct ps3_dma_region *r, unsigned long phys_addr, 513 unsigned long len, struct dma_chunk **c_out, u64 iopte_flag) 514 { 515 int result; 516 struct dma_chunk *c; 517 518 c = kzalloc(sizeof(*c), GFP_ATOMIC); 519 if (!c) { 520 result = -ENOMEM; 521 goto fail_alloc; 522 } 523 524 c->region = r; 525 c->lpar_addr = ps3_mm_phys_to_lpar(phys_addr); 526 c->bus_addr = dma_sb_lpar_to_bus(r, c->lpar_addr); 527 c->len = len; 528 529 BUG_ON(iopte_flag != 0xf800000000000000UL); 530 result = lv1_map_device_dma_region(c->region->dev->bus_id, 531 c->region->dev->dev_id, c->lpar_addr, 532 c->bus_addr, c->len, iopte_flag); 533 if (result) { 534 DBG("%s:%d: lv1_map_device_dma_region failed: %s\n", 535 __func__, __LINE__, ps3_result(result)); 536 goto fail_map; 537 } 538 539 list_add(&c->link, &r->chunk_list.head); 540 541 *c_out = c; 542 return 0; 543 544 fail_map: 545 kfree(c); 546 fail_alloc: 547 *c_out = NULL; 548 DBG(" <- %s:%d\n", __func__, __LINE__); 549 return result; 550 } 551 552 static int dma_ioc0_map_pages(struct ps3_dma_region *r, unsigned long phys_addr, 553 unsigned long len, struct dma_chunk **c_out, 554 u64 iopte_flag) 555 { 556 int result; 557 struct dma_chunk *c, *last; 558 int iopage, pages; 559 unsigned long offset; 560 561 DBG(KERN_ERR "%s: phy=%#lx, lpar%#lx, len=%#lx\n", __func__, 562 phys_addr, ps3_mm_phys_to_lpar(phys_addr), len); 563 c = kzalloc(sizeof(*c), GFP_ATOMIC); 564 if (!c) { 565 result = -ENOMEM; 566 goto fail_alloc; 567 } 568 569 c->region = r; 570 c->len = len; 571 c->lpar_addr = ps3_mm_phys_to_lpar(phys_addr); 572 /* allocate IO address */ 573 if (list_empty(&r->chunk_list.head)) { 574 /* first one */ 575 c->bus_addr = r->bus_addr; 576 } else { 577 /* derive from last bus addr*/ 578 last = list_entry(r->chunk_list.head.next, 579 struct dma_chunk, link); 580 c->bus_addr = last->bus_addr + last->len; 581 DBG("%s: last bus=%#lx, len=%#lx\n", __func__, 582 last->bus_addr, last->len); 583 } 584 585 /* FIXME: check whether length exceeds region size */ 586 587 /* build ioptes for the area */ 588 pages = len >> r->page_size; 589 DBG("%s: pgsize=%#x len=%#lx pages=%#x iopteflag=%#llx\n", __func__, 590 r->page_size, r->len, pages, iopte_flag); 591 for (iopage = 0; iopage < pages; iopage++) { 592 offset = (1 << r->page_size) * iopage; 593 result = lv1_put_iopte(0, 594 c->bus_addr + offset, 595 c->lpar_addr + offset, 596 r->ioid, 597 iopte_flag); 598 if (result) { 599 pr_warn("%s:%d: lv1_put_iopte failed: %s\n", 600 __func__, __LINE__, ps3_result(result)); 601 goto fail_map; 602 } 603 DBG("%s: pg=%d bus=%#lx, lpar=%#lx, ioid=%#x\n", __func__, 604 iopage, c->bus_addr + offset, c->lpar_addr + offset, 605 r->ioid); 606 } 607 608 /* be sure that last allocated one is inserted at head */ 609 list_add(&c->link, &r->chunk_list.head); 610 611 *c_out = c; 612 DBG("%s: end\n", __func__); 613 return 0; 614 615 fail_map: 616 for (iopage--; 0 <= iopage; iopage--) { 617 lv1_put_iopte(0, 618 c->bus_addr + offset, 619 c->lpar_addr + offset, 620 r->ioid, 621 0); 622 } 623 kfree(c); 624 fail_alloc: 625 *c_out = NULL; 626 return result; 627 } 628 629 /** 630 * dma_sb_region_create - Create a device dma region. 631 * @r: Pointer to a struct ps3_dma_region. 632 * 633 * This is the lowest level dma region create routine, and is the one that 634 * will make the HV call to create the region. 635 */ 636 637 static int dma_sb_region_create(struct ps3_dma_region *r) 638 { 639 int result; 640 u64 bus_addr; 641 642 DBG(" -> %s:%d:\n", __func__, __LINE__); 643 644 BUG_ON(!r); 645 646 if (!r->dev->bus_id) { 647 pr_info("%s:%d: %llu:%llu no dma\n", __func__, __LINE__, 648 r->dev->bus_id, r->dev->dev_id); 649 return 0; 650 } 651 652 DBG("%s:%u: len = 0x%lx, page_size = %u, offset = 0x%lx\n", __func__, 653 __LINE__, r->len, r->page_size, r->offset); 654 655 BUG_ON(!r->len); 656 BUG_ON(!r->page_size); 657 BUG_ON(!r->region_ops); 658 659 INIT_LIST_HEAD(&r->chunk_list.head); 660 spin_lock_init(&r->chunk_list.lock); 661 662 result = lv1_allocate_device_dma_region(r->dev->bus_id, r->dev->dev_id, 663 roundup_pow_of_two(r->len), r->page_size, r->region_type, 664 &bus_addr); 665 r->bus_addr = bus_addr; 666 667 if (result) { 668 DBG("%s:%d: lv1_allocate_device_dma_region failed: %s\n", 669 __func__, __LINE__, ps3_result(result)); 670 r->len = r->bus_addr = 0; 671 } 672 673 return result; 674 } 675 676 static int dma_ioc0_region_create(struct ps3_dma_region *r) 677 { 678 int result; 679 u64 bus_addr; 680 681 INIT_LIST_HEAD(&r->chunk_list.head); 682 spin_lock_init(&r->chunk_list.lock); 683 684 result = lv1_allocate_io_segment(0, 685 r->len, 686 r->page_size, 687 &bus_addr); 688 r->bus_addr = bus_addr; 689 if (result) { 690 DBG("%s:%d: lv1_allocate_io_segment failed: %s\n", 691 __func__, __LINE__, ps3_result(result)); 692 r->len = r->bus_addr = 0; 693 } 694 DBG("%s: len=%#lx, pg=%d, bus=%#lx\n", __func__, 695 r->len, r->page_size, r->bus_addr); 696 return result; 697 } 698 699 /** 700 * dma_region_free - Free a device dma region. 701 * @r: Pointer to a struct ps3_dma_region. 702 * 703 * This is the lowest level dma region free routine, and is the one that 704 * will make the HV call to free the region. 705 */ 706 707 static int dma_sb_region_free(struct ps3_dma_region *r) 708 { 709 int result; 710 struct dma_chunk *c; 711 struct dma_chunk *tmp; 712 713 BUG_ON(!r); 714 715 if (!r->dev->bus_id) { 716 pr_info("%s:%d: %llu:%llu no dma\n", __func__, __LINE__, 717 r->dev->bus_id, r->dev->dev_id); 718 return 0; 719 } 720 721 list_for_each_entry_safe(c, tmp, &r->chunk_list.head, link) { 722 list_del(&c->link); 723 dma_sb_free_chunk(c); 724 } 725 726 result = lv1_free_device_dma_region(r->dev->bus_id, r->dev->dev_id, 727 r->bus_addr); 728 729 if (result) 730 DBG("%s:%d: lv1_free_device_dma_region failed: %s\n", 731 __func__, __LINE__, ps3_result(result)); 732 733 r->bus_addr = 0; 734 735 return result; 736 } 737 738 static int dma_ioc0_region_free(struct ps3_dma_region *r) 739 { 740 int result; 741 struct dma_chunk *c, *n; 742 743 DBG("%s: start\n", __func__); 744 list_for_each_entry_safe(c, n, &r->chunk_list.head, link) { 745 list_del(&c->link); 746 dma_ioc0_free_chunk(c); 747 } 748 749 result = lv1_release_io_segment(0, r->bus_addr); 750 751 if (result) 752 DBG("%s:%d: lv1_free_device_dma_region failed: %s\n", 753 __func__, __LINE__, ps3_result(result)); 754 755 r->bus_addr = 0; 756 DBG("%s: end\n", __func__); 757 758 return result; 759 } 760 761 /** 762 * dma_sb_map_area - Map an area of memory into a device dma region. 763 * @r: Pointer to a struct ps3_dma_region. 764 * @virt_addr: Starting virtual address of the area to map. 765 * @len: Length in bytes of the area to map. 766 * @bus_addr: A pointer to return the starting ioc bus address of the area to 767 * map. 768 * 769 * This is the common dma mapping routine. 770 */ 771 772 static int dma_sb_map_area(struct ps3_dma_region *r, unsigned long virt_addr, 773 unsigned long len, dma_addr_t *bus_addr, 774 u64 iopte_flag) 775 { 776 int result; 777 unsigned long flags; 778 struct dma_chunk *c; 779 unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr) 780 : virt_addr; 781 unsigned long aligned_phys = ALIGN_DOWN(phys_addr, 1 << r->page_size); 782 unsigned long aligned_len = ALIGN(len + phys_addr - aligned_phys, 783 1 << r->page_size); 784 *bus_addr = dma_sb_lpar_to_bus(r, ps3_mm_phys_to_lpar(phys_addr)); 785 786 if (!USE_DYNAMIC_DMA) { 787 unsigned long lpar_addr = ps3_mm_phys_to_lpar(phys_addr); 788 DBG(" -> %s:%d\n", __func__, __LINE__); 789 DBG("%s:%d virt_addr %lxh\n", __func__, __LINE__, 790 virt_addr); 791 DBG("%s:%d phys_addr %lxh\n", __func__, __LINE__, 792 phys_addr); 793 DBG("%s:%d lpar_addr %lxh\n", __func__, __LINE__, 794 lpar_addr); 795 DBG("%s:%d len %lxh\n", __func__, __LINE__, len); 796 DBG("%s:%d bus_addr %llxh (%lxh)\n", __func__, __LINE__, 797 *bus_addr, len); 798 } 799 800 spin_lock_irqsave(&r->chunk_list.lock, flags); 801 c = dma_find_chunk(r, *bus_addr, len); 802 803 if (c) { 804 DBG("%s:%d: reusing mapped chunk", __func__, __LINE__); 805 dma_dump_chunk(c); 806 c->usage_count++; 807 spin_unlock_irqrestore(&r->chunk_list.lock, flags); 808 return 0; 809 } 810 811 result = dma_sb_map_pages(r, aligned_phys, aligned_len, &c, iopte_flag); 812 813 if (result) { 814 *bus_addr = 0; 815 DBG("%s:%d: dma_sb_map_pages failed (%d)\n", 816 __func__, __LINE__, result); 817 spin_unlock_irqrestore(&r->chunk_list.lock, flags); 818 return result; 819 } 820 821 c->usage_count = 1; 822 823 spin_unlock_irqrestore(&r->chunk_list.lock, flags); 824 return result; 825 } 826 827 static int dma_ioc0_map_area(struct ps3_dma_region *r, unsigned long virt_addr, 828 unsigned long len, dma_addr_t *bus_addr, 829 u64 iopte_flag) 830 { 831 int result; 832 unsigned long flags; 833 struct dma_chunk *c; 834 unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr) 835 : virt_addr; 836 unsigned long aligned_phys = ALIGN_DOWN(phys_addr, 1 << r->page_size); 837 unsigned long aligned_len = ALIGN(len + phys_addr - aligned_phys, 838 1 << r->page_size); 839 840 DBG(KERN_ERR "%s: vaddr=%#lx, len=%#lx\n", __func__, 841 virt_addr, len); 842 DBG(KERN_ERR "%s: ph=%#lx a_ph=%#lx a_l=%#lx\n", __func__, 843 phys_addr, aligned_phys, aligned_len); 844 845 spin_lock_irqsave(&r->chunk_list.lock, flags); 846 c = dma_find_chunk_lpar(r, ps3_mm_phys_to_lpar(phys_addr), len); 847 848 if (c) { 849 /* FIXME */ 850 BUG(); 851 *bus_addr = c->bus_addr + phys_addr - aligned_phys; 852 c->usage_count++; 853 spin_unlock_irqrestore(&r->chunk_list.lock, flags); 854 return 0; 855 } 856 857 result = dma_ioc0_map_pages(r, aligned_phys, aligned_len, &c, 858 iopte_flag); 859 860 if (result) { 861 *bus_addr = 0; 862 DBG("%s:%d: dma_ioc0_map_pages failed (%d)\n", 863 __func__, __LINE__, result); 864 spin_unlock_irqrestore(&r->chunk_list.lock, flags); 865 return result; 866 } 867 *bus_addr = c->bus_addr + phys_addr - aligned_phys; 868 DBG("%s: va=%#lx pa=%#lx a_pa=%#lx bus=%#llx\n", __func__, 869 virt_addr, phys_addr, aligned_phys, *bus_addr); 870 c->usage_count = 1; 871 872 spin_unlock_irqrestore(&r->chunk_list.lock, flags); 873 return result; 874 } 875 876 /** 877 * dma_sb_unmap_area - Unmap an area of memory from a device dma region. 878 * @r: Pointer to a struct ps3_dma_region. 879 * @bus_addr: The starting ioc bus address of the area to unmap. 880 * @len: Length in bytes of the area to unmap. 881 * 882 * This is the common dma unmap routine. 883 */ 884 885 static int dma_sb_unmap_area(struct ps3_dma_region *r, dma_addr_t bus_addr, 886 unsigned long len) 887 { 888 unsigned long flags; 889 struct dma_chunk *c; 890 891 spin_lock_irqsave(&r->chunk_list.lock, flags); 892 c = dma_find_chunk(r, bus_addr, len); 893 894 if (!c) { 895 unsigned long aligned_bus = ALIGN_DOWN(bus_addr, 896 1 << r->page_size); 897 unsigned long aligned_len = ALIGN(len + bus_addr 898 - aligned_bus, 1 << r->page_size); 899 DBG("%s:%d: not found: bus_addr %llxh\n", 900 __func__, __LINE__, bus_addr); 901 DBG("%s:%d: not found: len %lxh\n", 902 __func__, __LINE__, len); 903 DBG("%s:%d: not found: aligned_bus %lxh\n", 904 __func__, __LINE__, aligned_bus); 905 DBG("%s:%d: not found: aligned_len %lxh\n", 906 __func__, __LINE__, aligned_len); 907 BUG(); 908 } 909 910 c->usage_count--; 911 912 if (!c->usage_count) { 913 list_del(&c->link); 914 dma_sb_free_chunk(c); 915 } 916 917 spin_unlock_irqrestore(&r->chunk_list.lock, flags); 918 return 0; 919 } 920 921 static int dma_ioc0_unmap_area(struct ps3_dma_region *r, 922 dma_addr_t bus_addr, unsigned long len) 923 { 924 unsigned long flags; 925 struct dma_chunk *c; 926 927 DBG("%s: start a=%#llx l=%#lx\n", __func__, bus_addr, len); 928 spin_lock_irqsave(&r->chunk_list.lock, flags); 929 c = dma_find_chunk(r, bus_addr, len); 930 931 if (!c) { 932 unsigned long aligned_bus = ALIGN_DOWN(bus_addr, 933 1 << r->page_size); 934 unsigned long aligned_len = ALIGN(len + bus_addr 935 - aligned_bus, 936 1 << r->page_size); 937 DBG("%s:%d: not found: bus_addr %llxh\n", 938 __func__, __LINE__, bus_addr); 939 DBG("%s:%d: not found: len %lxh\n", 940 __func__, __LINE__, len); 941 DBG("%s:%d: not found: aligned_bus %lxh\n", 942 __func__, __LINE__, aligned_bus); 943 DBG("%s:%d: not found: aligned_len %lxh\n", 944 __func__, __LINE__, aligned_len); 945 BUG(); 946 } 947 948 c->usage_count--; 949 950 if (!c->usage_count) { 951 list_del(&c->link); 952 dma_ioc0_free_chunk(c); 953 } 954 955 spin_unlock_irqrestore(&r->chunk_list.lock, flags); 956 DBG("%s: end\n", __func__); 957 return 0; 958 } 959 960 /** 961 * dma_sb_region_create_linear - Setup a linear dma mapping for a device. 962 * @r: Pointer to a struct ps3_dma_region. 963 * 964 * This routine creates an HV dma region for the device and maps all available 965 * ram into the io controller bus address space. 966 */ 967 968 static int dma_sb_region_create_linear(struct ps3_dma_region *r) 969 { 970 int result; 971 unsigned long virt_addr, len; 972 dma_addr_t tmp; 973 974 if (r->len > 16*1024*1024) { /* FIXME: need proper fix */ 975 /* force 16M dma pages for linear mapping */ 976 if (r->page_size != PS3_DMA_16M) { 977 pr_info("%s:%d: forcing 16M pages for linear map\n", 978 __func__, __LINE__); 979 r->page_size = PS3_DMA_16M; 980 r->len = ALIGN(r->len, 1 << r->page_size); 981 } 982 } 983 984 result = dma_sb_region_create(r); 985 BUG_ON(result); 986 987 if (r->offset < map.rm.size) { 988 /* Map (part of) 1st RAM chunk */ 989 virt_addr = map.rm.base + r->offset; 990 len = map.rm.size - r->offset; 991 if (len > r->len) 992 len = r->len; 993 result = dma_sb_map_area(r, virt_addr, len, &tmp, 994 CBE_IOPTE_PP_W | CBE_IOPTE_PP_R | CBE_IOPTE_SO_RW | 995 CBE_IOPTE_M); 996 BUG_ON(result); 997 } 998 999 if (r->offset + r->len > map.rm.size) { 1000 /* Map (part of) 2nd RAM chunk */ 1001 virt_addr = map.rm.size; 1002 len = r->len; 1003 if (r->offset >= map.rm.size) 1004 virt_addr += r->offset - map.rm.size; 1005 else 1006 len -= map.rm.size - r->offset; 1007 result = dma_sb_map_area(r, virt_addr, len, &tmp, 1008 CBE_IOPTE_PP_W | CBE_IOPTE_PP_R | CBE_IOPTE_SO_RW | 1009 CBE_IOPTE_M); 1010 BUG_ON(result); 1011 } 1012 1013 return result; 1014 } 1015 1016 /** 1017 * dma_sb_region_free_linear - Free a linear dma mapping for a device. 1018 * @r: Pointer to a struct ps3_dma_region. 1019 * 1020 * This routine will unmap all mapped areas and free the HV dma region. 1021 */ 1022 1023 static int dma_sb_region_free_linear(struct ps3_dma_region *r) 1024 { 1025 int result; 1026 dma_addr_t bus_addr; 1027 unsigned long len, lpar_addr; 1028 1029 if (r->offset < map.rm.size) { 1030 /* Unmap (part of) 1st RAM chunk */ 1031 lpar_addr = map.rm.base + r->offset; 1032 len = map.rm.size - r->offset; 1033 if (len > r->len) 1034 len = r->len; 1035 bus_addr = dma_sb_lpar_to_bus(r, lpar_addr); 1036 result = dma_sb_unmap_area(r, bus_addr, len); 1037 BUG_ON(result); 1038 } 1039 1040 if (r->offset + r->len > map.rm.size) { 1041 /* Unmap (part of) 2nd RAM chunk */ 1042 lpar_addr = map.r1.base; 1043 len = r->len; 1044 if (r->offset >= map.rm.size) 1045 lpar_addr += r->offset - map.rm.size; 1046 else 1047 len -= map.rm.size - r->offset; 1048 bus_addr = dma_sb_lpar_to_bus(r, lpar_addr); 1049 result = dma_sb_unmap_area(r, bus_addr, len); 1050 BUG_ON(result); 1051 } 1052 1053 result = dma_sb_region_free(r); 1054 BUG_ON(result); 1055 1056 return result; 1057 } 1058 1059 /** 1060 * dma_sb_map_area_linear - Map an area of memory into a device dma region. 1061 * @r: Pointer to a struct ps3_dma_region. 1062 * @virt_addr: Starting virtual address of the area to map. 1063 * @len: Length in bytes of the area to map. 1064 * @bus_addr: A pointer to return the starting ioc bus address of the area to 1065 * map. 1066 * 1067 * This routine just returns the corresponding bus address. Actual mapping 1068 * occurs in dma_region_create_linear(). 1069 */ 1070 1071 static int dma_sb_map_area_linear(struct ps3_dma_region *r, 1072 unsigned long virt_addr, unsigned long len, dma_addr_t *bus_addr, 1073 u64 iopte_flag) 1074 { 1075 unsigned long phys_addr = is_kernel_addr(virt_addr) ? __pa(virt_addr) 1076 : virt_addr; 1077 *bus_addr = dma_sb_lpar_to_bus(r, ps3_mm_phys_to_lpar(phys_addr)); 1078 return 0; 1079 } 1080 1081 /** 1082 * dma_unmap_area_linear - Unmap an area of memory from a device dma region. 1083 * @r: Pointer to a struct ps3_dma_region. 1084 * @bus_addr: The starting ioc bus address of the area to unmap. 1085 * @len: Length in bytes of the area to unmap. 1086 * 1087 * This routine does nothing. Unmapping occurs in dma_sb_region_free_linear(). 1088 */ 1089 1090 static int dma_sb_unmap_area_linear(struct ps3_dma_region *r, 1091 dma_addr_t bus_addr, unsigned long len) 1092 { 1093 return 0; 1094 }; 1095 1096 static const struct ps3_dma_region_ops ps3_dma_sb_region_ops = { 1097 .create = dma_sb_region_create, 1098 .free = dma_sb_region_free, 1099 .map = dma_sb_map_area, 1100 .unmap = dma_sb_unmap_area 1101 }; 1102 1103 static const struct ps3_dma_region_ops ps3_dma_sb_region_linear_ops = { 1104 .create = dma_sb_region_create_linear, 1105 .free = dma_sb_region_free_linear, 1106 .map = dma_sb_map_area_linear, 1107 .unmap = dma_sb_unmap_area_linear 1108 }; 1109 1110 static const struct ps3_dma_region_ops ps3_dma_ioc0_region_ops = { 1111 .create = dma_ioc0_region_create, 1112 .free = dma_ioc0_region_free, 1113 .map = dma_ioc0_map_area, 1114 .unmap = dma_ioc0_unmap_area 1115 }; 1116 1117 int ps3_dma_region_init(struct ps3_system_bus_device *dev, 1118 struct ps3_dma_region *r, enum ps3_dma_page_size page_size, 1119 enum ps3_dma_region_type region_type, void *addr, unsigned long len) 1120 { 1121 unsigned long lpar_addr; 1122 int result; 1123 1124 lpar_addr = addr ? ps3_mm_phys_to_lpar(__pa(addr)) : 0; 1125 1126 r->dev = dev; 1127 r->page_size = page_size; 1128 r->region_type = region_type; 1129 r->offset = lpar_addr; 1130 if (r->offset >= map.rm.size) 1131 r->offset -= map.r1.offset; 1132 r->len = len ? len : ALIGN(map.total, 1 << r->page_size); 1133 1134 dev->core.dma_mask = &r->dma_mask; 1135 1136 result = dma_set_mask_and_coherent(&dev->core, DMA_BIT_MASK(32)); 1137 1138 if (result < 0) { 1139 dev_err(&dev->core, "%s:%d: dma_set_mask_and_coherent failed: %d\n", 1140 __func__, __LINE__, result); 1141 return result; 1142 } 1143 1144 switch (dev->dev_type) { 1145 case PS3_DEVICE_TYPE_SB: 1146 r->region_ops = (USE_DYNAMIC_DMA) 1147 ? &ps3_dma_sb_region_ops 1148 : &ps3_dma_sb_region_linear_ops; 1149 break; 1150 case PS3_DEVICE_TYPE_IOC0: 1151 r->region_ops = &ps3_dma_ioc0_region_ops; 1152 break; 1153 default: 1154 BUG(); 1155 return -EINVAL; 1156 } 1157 return 0; 1158 } 1159 EXPORT_SYMBOL(ps3_dma_region_init); 1160 1161 int ps3_dma_region_create(struct ps3_dma_region *r) 1162 { 1163 BUG_ON(!r); 1164 BUG_ON(!r->region_ops); 1165 BUG_ON(!r->region_ops->create); 1166 return r->region_ops->create(r); 1167 } 1168 EXPORT_SYMBOL(ps3_dma_region_create); 1169 1170 int ps3_dma_region_free(struct ps3_dma_region *r) 1171 { 1172 BUG_ON(!r); 1173 BUG_ON(!r->region_ops); 1174 BUG_ON(!r->region_ops->free); 1175 return r->region_ops->free(r); 1176 } 1177 EXPORT_SYMBOL(ps3_dma_region_free); 1178 1179 int ps3_dma_map(struct ps3_dma_region *r, unsigned long virt_addr, 1180 unsigned long len, dma_addr_t *bus_addr, 1181 u64 iopte_flag) 1182 { 1183 return r->region_ops->map(r, virt_addr, len, bus_addr, iopte_flag); 1184 } 1185 1186 int ps3_dma_unmap(struct ps3_dma_region *r, dma_addr_t bus_addr, 1187 unsigned long len) 1188 { 1189 return r->region_ops->unmap(r, bus_addr, len); 1190 } 1191 1192 /*============================================================================*/ 1193 /* system startup routines */ 1194 /*============================================================================*/ 1195 1196 /** 1197 * ps3_mm_init - initialize the address space state variables 1198 */ 1199 1200 void __init ps3_mm_init(void) 1201 { 1202 int result; 1203 1204 DBG(" -> %s:%d\n", __func__, __LINE__); 1205 1206 result = ps3_repository_read_mm_info(&map.rm.base, &map.rm.size, 1207 &map.total); 1208 1209 if (result) 1210 panic("ps3_repository_read_mm_info() failed"); 1211 1212 map.rm.offset = map.rm.base; 1213 map.vas_id = map.htab_size = 0; 1214 1215 /* this implementation assumes map.rm.base is zero */ 1216 1217 BUG_ON(map.rm.base); 1218 BUG_ON(!map.rm.size); 1219 1220 /* Check if we got the highmem region from an earlier boot step */ 1221 1222 if (ps3_mm_get_repository_highmem(&map.r1)) { 1223 result = ps3_mm_region_create(&map.r1, map.total - map.rm.size); 1224 1225 if (!result) 1226 ps3_mm_set_repository_highmem(&map.r1); 1227 } 1228 1229 /* correct map.total for the real total amount of memory we use */ 1230 map.total = map.rm.size + map.r1.size; 1231 1232 if (!map.r1.size) { 1233 DBG("%s:%d: No highmem region found\n", __func__, __LINE__); 1234 } else { 1235 DBG("%s:%d: Adding highmem region: %llxh %llxh\n", 1236 __func__, __LINE__, map.rm.size, 1237 map.total - map.rm.size); 1238 memblock_add(map.rm.size, map.total - map.rm.size); 1239 } 1240 1241 DBG(" <- %s:%d\n", __func__, __LINE__); 1242 } 1243 1244 /** 1245 * ps3_mm_shutdown - final cleanup of address space 1246 */ 1247 1248 void ps3_mm_shutdown(void) 1249 { 1250 ps3_mm_region_destroy(&map.r1); 1251 } 1252