1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/memblock.h> 3 #include <linux/gfp.h> 4 #include <linux/export.h> 5 #include <linux/spinlock.h> 6 #include <linux/slab.h> 7 #include <linux/types.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/vmalloc.h> 10 #include <linux/swiotlb.h> 11 12 #include <xen/xen.h> 13 #include <xen/interface/memory.h> 14 #include <xen/page.h> 15 #include <xen/swiotlb-xen.h> 16 17 #include <asm/cacheflush.h> 18 #include <asm/xen/hypercall.h> 19 #include <asm/xen/interface.h> 20 21 struct xen_p2m_entry { 22 unsigned long pfn; 23 unsigned long mfn; 24 unsigned long nr_pages; 25 struct rb_node rbnode_phys; 26 }; 27 28 static rwlock_t p2m_lock; 29 struct rb_root phys_to_mach = RB_ROOT; 30 EXPORT_SYMBOL_GPL(phys_to_mach); 31 32 static int xen_add_phys_to_mach_entry(struct xen_p2m_entry *new) 33 { 34 struct rb_node **link = &phys_to_mach.rb_node; 35 struct rb_node *parent = NULL; 36 struct xen_p2m_entry *entry; 37 int rc = 0; 38 39 while (*link) { 40 parent = *link; 41 entry = rb_entry(parent, struct xen_p2m_entry, rbnode_phys); 42 43 if (new->pfn == entry->pfn) 44 goto err_out; 45 46 if (new->pfn < entry->pfn) 47 link = &(*link)->rb_left; 48 else 49 link = &(*link)->rb_right; 50 } 51 rb_link_node(&new->rbnode_phys, parent, link); 52 rb_insert_color(&new->rbnode_phys, &phys_to_mach); 53 goto out; 54 55 err_out: 56 rc = -EINVAL; 57 pr_warn("%s: cannot add pfn=%pa -> mfn=%pa: pfn=%pa -> mfn=%pa already exists\n", 58 __func__, &new->pfn, &new->mfn, &entry->pfn, &entry->mfn); 59 out: 60 return rc; 61 } 62 63 unsigned long __pfn_to_mfn(unsigned long pfn) 64 { 65 struct rb_node *n = phys_to_mach.rb_node; 66 struct xen_p2m_entry *entry; 67 unsigned long irqflags; 68 69 read_lock_irqsave(&p2m_lock, irqflags); 70 while (n) { 71 entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); 72 if (entry->pfn <= pfn && 73 entry->pfn + entry->nr_pages > pfn) { 74 unsigned long mfn = entry->mfn + (pfn - entry->pfn); 75 read_unlock_irqrestore(&p2m_lock, irqflags); 76 return mfn; 77 } 78 if (pfn < entry->pfn) 79 n = n->rb_left; 80 else 81 n = n->rb_right; 82 } 83 read_unlock_irqrestore(&p2m_lock, irqflags); 84 85 return INVALID_P2M_ENTRY; 86 } 87 EXPORT_SYMBOL_GPL(__pfn_to_mfn); 88 89 int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, 90 struct gnttab_map_grant_ref *kmap_ops, 91 struct page **pages, unsigned int count) 92 { 93 int i; 94 95 for (i = 0; i < count; i++) { 96 if (map_ops[i].status) 97 continue; 98 if (unlikely(!set_phys_to_machine(map_ops[i].host_addr >> XEN_PAGE_SHIFT, 99 map_ops[i].dev_bus_addr >> XEN_PAGE_SHIFT))) { 100 return -ENOMEM; 101 } 102 } 103 104 return 0; 105 } 106 EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); 107 108 int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, 109 struct gnttab_unmap_grant_ref *kunmap_ops, 110 struct page **pages, unsigned int count) 111 { 112 int i; 113 114 for (i = 0; i < count; i++) { 115 set_phys_to_machine(unmap_ops[i].host_addr >> XEN_PAGE_SHIFT, 116 INVALID_P2M_ENTRY); 117 } 118 119 return 0; 120 } 121 EXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping); 122 123 bool __set_phys_to_machine_multi(unsigned long pfn, 124 unsigned long mfn, unsigned long nr_pages) 125 { 126 int rc; 127 unsigned long irqflags; 128 struct xen_p2m_entry *p2m_entry; 129 struct rb_node *n = phys_to_mach.rb_node; 130 131 if (mfn == INVALID_P2M_ENTRY) { 132 write_lock_irqsave(&p2m_lock, irqflags); 133 while (n) { 134 p2m_entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); 135 if (p2m_entry->pfn <= pfn && 136 p2m_entry->pfn + p2m_entry->nr_pages > pfn) { 137 rb_erase(&p2m_entry->rbnode_phys, &phys_to_mach); 138 write_unlock_irqrestore(&p2m_lock, irqflags); 139 kfree(p2m_entry); 140 return true; 141 } 142 if (pfn < p2m_entry->pfn) 143 n = n->rb_left; 144 else 145 n = n->rb_right; 146 } 147 write_unlock_irqrestore(&p2m_lock, irqflags); 148 return true; 149 } 150 151 p2m_entry = kzalloc(sizeof(*p2m_entry), GFP_NOWAIT); 152 if (!p2m_entry) 153 return false; 154 155 p2m_entry->pfn = pfn; 156 p2m_entry->nr_pages = nr_pages; 157 p2m_entry->mfn = mfn; 158 159 write_lock_irqsave(&p2m_lock, irqflags); 160 rc = xen_add_phys_to_mach_entry(p2m_entry); 161 if (rc < 0) { 162 write_unlock_irqrestore(&p2m_lock, irqflags); 163 kfree(p2m_entry); 164 return false; 165 } 166 write_unlock_irqrestore(&p2m_lock, irqflags); 167 return true; 168 } 169 EXPORT_SYMBOL_GPL(__set_phys_to_machine_multi); 170 171 bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn) 172 { 173 return __set_phys_to_machine_multi(pfn, mfn, 1); 174 } 175 EXPORT_SYMBOL_GPL(__set_phys_to_machine); 176 177 static int p2m_init(void) 178 { 179 rwlock_init(&p2m_lock); 180 return 0; 181 } 182 arch_initcall(p2m_init); 183