1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 257c8a661SMike Rapoport #include <linux/memblock.h> 34a19138cSStefano Stabellini #include <linux/gfp.h> 44a19138cSStefano Stabellini #include <linux/export.h> 5a494ee6cSSebastian Andrzej Siewior #include <linux/spinlock.h> 64a19138cSStefano Stabellini #include <linux/slab.h> 74a19138cSStefano Stabellini #include <linux/types.h> 84a19138cSStefano Stabellini #include <linux/dma-mapping.h> 94a19138cSStefano Stabellini #include <linux/vmalloc.h> 104a19138cSStefano Stabellini #include <linux/swiotlb.h> 114a19138cSStefano Stabellini 124a19138cSStefano Stabellini #include <xen/xen.h> 134a19138cSStefano Stabellini #include <xen/interface/memory.h> 14a9fd60e2SJulien Grall #include <xen/page.h> 154a19138cSStefano Stabellini #include <xen/swiotlb-xen.h> 164a19138cSStefano Stabellini 174a19138cSStefano Stabellini #include <asm/cacheflush.h> 184a19138cSStefano Stabellini #include <asm/xen/hypercall.h> 194a19138cSStefano Stabellini #include <asm/xen/interface.h> 204a19138cSStefano Stabellini 214a19138cSStefano Stabellini struct xen_p2m_entry { 224a19138cSStefano Stabellini unsigned long pfn; 234a19138cSStefano Stabellini unsigned long mfn; 244a19138cSStefano Stabellini unsigned long nr_pages; 254a19138cSStefano Stabellini struct rb_node rbnode_phys; 264a19138cSStefano Stabellini }; 274a19138cSStefano Stabellini 28f9c7ec16SStefano Stabellini static rwlock_t p2m_lock; 294a19138cSStefano Stabellini struct rb_root phys_to_mach = RB_ROOT; 30c8999a88SJosh Boyer EXPORT_SYMBOL_GPL(phys_to_mach); 314a19138cSStefano Stabellini 324a19138cSStefano Stabellini static int xen_add_phys_to_mach_entry(struct xen_p2m_entry *new) 334a19138cSStefano Stabellini { 344a19138cSStefano Stabellini struct rb_node **link = &phys_to_mach.rb_node; 354a19138cSStefano Stabellini struct rb_node *parent = NULL; 364a19138cSStefano Stabellini struct xen_p2m_entry *entry; 374a19138cSStefano Stabellini int rc = 0; 384a19138cSStefano Stabellini 394a19138cSStefano Stabellini while (*link) { 404a19138cSStefano Stabellini parent = *link; 414a19138cSStefano Stabellini entry = rb_entry(parent, struct xen_p2m_entry, rbnode_phys); 424a19138cSStefano Stabellini 434a19138cSStefano Stabellini if (new->pfn == entry->pfn) 444a19138cSStefano Stabellini goto err_out; 454a19138cSStefano Stabellini 464a19138cSStefano Stabellini if (new->pfn < entry->pfn) 474a19138cSStefano Stabellini link = &(*link)->rb_left; 484a19138cSStefano Stabellini else 494a19138cSStefano Stabellini link = &(*link)->rb_right; 504a19138cSStefano Stabellini } 514a19138cSStefano Stabellini rb_link_node(&new->rbnode_phys, parent, link); 524a19138cSStefano Stabellini rb_insert_color(&new->rbnode_phys, &phys_to_mach); 534a19138cSStefano Stabellini goto out; 544a19138cSStefano Stabellini 554a19138cSStefano Stabellini err_out: 564a19138cSStefano Stabellini rc = -EINVAL; 574a19138cSStefano Stabellini pr_warn("%s: cannot add pfn=%pa -> mfn=%pa: pfn=%pa -> mfn=%pa already exists\n", 584a19138cSStefano Stabellini __func__, &new->pfn, &new->mfn, &entry->pfn, &entry->mfn); 594a19138cSStefano Stabellini out: 604a19138cSStefano Stabellini return rc; 614a19138cSStefano Stabellini } 624a19138cSStefano Stabellini 634a19138cSStefano Stabellini unsigned long __pfn_to_mfn(unsigned long pfn) 644a19138cSStefano Stabellini { 654a19138cSStefano Stabellini struct rb_node *n = phys_to_mach.rb_node; 664a19138cSStefano Stabellini struct xen_p2m_entry *entry; 674a19138cSStefano Stabellini unsigned long irqflags; 684a19138cSStefano Stabellini 694a19138cSStefano Stabellini read_lock_irqsave(&p2m_lock, irqflags); 704a19138cSStefano Stabellini while (n) { 714a19138cSStefano Stabellini entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); 724a19138cSStefano Stabellini if (entry->pfn <= pfn && 734a19138cSStefano Stabellini entry->pfn + entry->nr_pages > pfn) { 74fe846979SHillf Danton unsigned long mfn = entry->mfn + (pfn - entry->pfn); 754a19138cSStefano Stabellini read_unlock_irqrestore(&p2m_lock, irqflags); 76fe846979SHillf Danton return mfn; 774a19138cSStefano Stabellini } 784a19138cSStefano Stabellini if (pfn < entry->pfn) 794a19138cSStefano Stabellini n = n->rb_left; 804a19138cSStefano Stabellini else 814a19138cSStefano Stabellini n = n->rb_right; 824a19138cSStefano Stabellini } 834a19138cSStefano Stabellini read_unlock_irqrestore(&p2m_lock, irqflags); 844a19138cSStefano Stabellini 854a19138cSStefano Stabellini return INVALID_P2M_ENTRY; 864a19138cSStefano Stabellini } 874a19138cSStefano Stabellini EXPORT_SYMBOL_GPL(__pfn_to_mfn); 884a19138cSStefano Stabellini 891429d46dSZoltan Kiss int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, 901429d46dSZoltan Kiss struct gnttab_map_grant_ref *kmap_ops, 911429d46dSZoltan Kiss struct page **pages, unsigned int count) 921429d46dSZoltan Kiss { 931429d46dSZoltan Kiss int i; 941429d46dSZoltan Kiss 951429d46dSZoltan Kiss for (i = 0; i < count; i++) { 96*8310b77bSJan Beulich struct gnttab_unmap_grant_ref unmap; 97*8310b77bSJan Beulich int rc; 98*8310b77bSJan Beulich 991429d46dSZoltan Kiss if (map_ops[i].status) 1001429d46dSZoltan Kiss continue; 101*8310b77bSJan Beulich if (likely(set_phys_to_machine(map_ops[i].host_addr >> XEN_PAGE_SHIFT, 102*8310b77bSJan Beulich map_ops[i].dev_bus_addr >> XEN_PAGE_SHIFT))) 103*8310b77bSJan Beulich continue; 104*8310b77bSJan Beulich 105*8310b77bSJan Beulich /* 106*8310b77bSJan Beulich * Signal an error for this slot. This in turn requires 107*8310b77bSJan Beulich * immediate unmapping. 108*8310b77bSJan Beulich */ 109*8310b77bSJan Beulich map_ops[i].status = GNTST_general_error; 110*8310b77bSJan Beulich unmap.host_addr = map_ops[i].host_addr, 111*8310b77bSJan Beulich unmap.handle = map_ops[i].handle; 112*8310b77bSJan Beulich map_ops[i].handle = ~0; 113*8310b77bSJan Beulich if (map_ops[i].flags & GNTMAP_device_map) 114*8310b77bSJan Beulich unmap.dev_bus_addr = map_ops[i].dev_bus_addr; 115*8310b77bSJan Beulich else 116*8310b77bSJan Beulich unmap.dev_bus_addr = 0; 117*8310b77bSJan Beulich 118*8310b77bSJan Beulich /* 119*8310b77bSJan Beulich * Pre-populate the status field, to be recognizable in 120*8310b77bSJan Beulich * the log message below. 121*8310b77bSJan Beulich */ 122*8310b77bSJan Beulich unmap.status = 1; 123*8310b77bSJan Beulich 124*8310b77bSJan Beulich rc = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, 125*8310b77bSJan Beulich &unmap, 1); 126*8310b77bSJan Beulich if (rc || unmap.status != GNTST_okay) 127*8310b77bSJan Beulich pr_err_once("gnttab unmap failed: rc=%d st=%d\n", 128*8310b77bSJan Beulich rc, unmap.status); 1291429d46dSZoltan Kiss } 1301429d46dSZoltan Kiss 1311429d46dSZoltan Kiss return 0; 1321429d46dSZoltan Kiss } 1331429d46dSZoltan Kiss EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); 1341429d46dSZoltan Kiss 1351429d46dSZoltan Kiss int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, 136853d0289SDavid Vrabel struct gnttab_unmap_grant_ref *kunmap_ops, 1371429d46dSZoltan Kiss struct page **pages, unsigned int count) 1381429d46dSZoltan Kiss { 1391429d46dSZoltan Kiss int i; 1401429d46dSZoltan Kiss 1411429d46dSZoltan Kiss for (i = 0; i < count; i++) { 1425ed5451dSJulien Grall set_phys_to_machine(unmap_ops[i].host_addr >> XEN_PAGE_SHIFT, 1431429d46dSZoltan Kiss INVALID_P2M_ENTRY); 1441429d46dSZoltan Kiss } 1451429d46dSZoltan Kiss 1461429d46dSZoltan Kiss return 0; 1471429d46dSZoltan Kiss } 1481429d46dSZoltan Kiss EXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping); 1491429d46dSZoltan Kiss 1504a19138cSStefano Stabellini bool __set_phys_to_machine_multi(unsigned long pfn, 1514a19138cSStefano Stabellini unsigned long mfn, unsigned long nr_pages) 1524a19138cSStefano Stabellini { 1534a19138cSStefano Stabellini int rc; 1544a19138cSStefano Stabellini unsigned long irqflags; 1554a19138cSStefano Stabellini struct xen_p2m_entry *p2m_entry; 1564a19138cSStefano Stabellini struct rb_node *n = phys_to_mach.rb_node; 1574a19138cSStefano Stabellini 1584a19138cSStefano Stabellini if (mfn == INVALID_P2M_ENTRY) { 1594a19138cSStefano Stabellini write_lock_irqsave(&p2m_lock, irqflags); 1604a19138cSStefano Stabellini while (n) { 1614a19138cSStefano Stabellini p2m_entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); 1624a19138cSStefano Stabellini if (p2m_entry->pfn <= pfn && 1634a19138cSStefano Stabellini p2m_entry->pfn + p2m_entry->nr_pages > pfn) { 1644a19138cSStefano Stabellini rb_erase(&p2m_entry->rbnode_phys, &phys_to_mach); 1654a19138cSStefano Stabellini write_unlock_irqrestore(&p2m_lock, irqflags); 1664a19138cSStefano Stabellini kfree(p2m_entry); 1674a19138cSStefano Stabellini return true; 1684a19138cSStefano Stabellini } 1694a19138cSStefano Stabellini if (pfn < p2m_entry->pfn) 1704a19138cSStefano Stabellini n = n->rb_left; 1714a19138cSStefano Stabellini else 1724a19138cSStefano Stabellini n = n->rb_right; 1734a19138cSStefano Stabellini } 1744a19138cSStefano Stabellini write_unlock_irqrestore(&p2m_lock, irqflags); 1754a19138cSStefano Stabellini return true; 1764a19138cSStefano Stabellini } 1774a19138cSStefano Stabellini 178031229b8SMarkus Elfring p2m_entry = kzalloc(sizeof(*p2m_entry), GFP_NOWAIT); 179d6bb4ec3SMarkus Elfring if (!p2m_entry) 1804a19138cSStefano Stabellini return false; 181d6bb4ec3SMarkus Elfring 1824a19138cSStefano Stabellini p2m_entry->pfn = pfn; 1834a19138cSStefano Stabellini p2m_entry->nr_pages = nr_pages; 1844a19138cSStefano Stabellini p2m_entry->mfn = mfn; 1854a19138cSStefano Stabellini 1864a19138cSStefano Stabellini write_lock_irqsave(&p2m_lock, irqflags); 18784a0a967SMarkus Elfring rc = xen_add_phys_to_mach_entry(p2m_entry); 18884a0a967SMarkus Elfring if (rc < 0) { 1894a19138cSStefano Stabellini write_unlock_irqrestore(&p2m_lock, irqflags); 190425f1cc2SHillf Danton kfree(p2m_entry); 1914a19138cSStefano Stabellini return false; 1924a19138cSStefano Stabellini } 1934a19138cSStefano Stabellini write_unlock_irqrestore(&p2m_lock, irqflags); 1944a19138cSStefano Stabellini return true; 1954a19138cSStefano Stabellini } 1964a19138cSStefano Stabellini EXPORT_SYMBOL_GPL(__set_phys_to_machine_multi); 1974a19138cSStefano Stabellini 1984a19138cSStefano Stabellini bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn) 1994a19138cSStefano Stabellini { 2004a19138cSStefano Stabellini return __set_phys_to_machine_multi(pfn, mfn, 1); 2014a19138cSStefano Stabellini } 2024a19138cSStefano Stabellini EXPORT_SYMBOL_GPL(__set_phys_to_machine); 2034a19138cSStefano Stabellini 204f9c7ec16SStefano Stabellini static int p2m_init(void) 2054a19138cSStefano Stabellini { 2064a19138cSStefano Stabellini rwlock_init(&p2m_lock); 2074a19138cSStefano Stabellini return 0; 2084a19138cSStefano Stabellini } 2094a19138cSStefano Stabellini arch_initcall(p2m_init); 210