1*e7218dd5SVikram Garhwal /*
2*e7218dd5SVikram Garhwal * Copyright (C) 2011 Citrix Ltd.
3*e7218dd5SVikram Garhwal *
4*e7218dd5SVikram Garhwal * This work is licensed under the terms of the GNU GPL, version 2. See
5*e7218dd5SVikram Garhwal * the COPYING file in the top-level directory.
6*e7218dd5SVikram Garhwal *
7*e7218dd5SVikram Garhwal * Contributions after 2012-01-13 are licensed under the terms of the
8*e7218dd5SVikram Garhwal * GNU GPL, version 2 or (at your option) any later version.
9*e7218dd5SVikram Garhwal */
10*e7218dd5SVikram Garhwal
11*e7218dd5SVikram Garhwal #include "qemu/osdep.h"
12*e7218dd5SVikram Garhwal #include "qemu/units.h"
13*e7218dd5SVikram Garhwal #include "qemu/error-report.h"
14*e7218dd5SVikram Garhwal
15*e7218dd5SVikram Garhwal #include <sys/resource.h>
16*e7218dd5SVikram Garhwal
17*e7218dd5SVikram Garhwal #include "hw/xen/xen_native.h"
18*e7218dd5SVikram Garhwal #include "qemu/bitmap.h"
19*e7218dd5SVikram Garhwal
20*e7218dd5SVikram Garhwal #include "sysemu/runstate.h"
21*e7218dd5SVikram Garhwal #include "sysemu/xen-mapcache.h"
22*e7218dd5SVikram Garhwal #include "trace.h"
23*e7218dd5SVikram Garhwal
24*e7218dd5SVikram Garhwal
25*e7218dd5SVikram Garhwal //#define MAPCACHE_DEBUG
26*e7218dd5SVikram Garhwal
27*e7218dd5SVikram Garhwal #ifdef MAPCACHE_DEBUG
28*e7218dd5SVikram Garhwal # define DPRINTF(fmt, ...) do { \
29*e7218dd5SVikram Garhwal fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
30*e7218dd5SVikram Garhwal } while (0)
31*e7218dd5SVikram Garhwal #else
32*e7218dd5SVikram Garhwal # define DPRINTF(fmt, ...) do { } while (0)
33*e7218dd5SVikram Garhwal #endif
34*e7218dd5SVikram Garhwal
35*e7218dd5SVikram Garhwal #if HOST_LONG_BITS == 32
36*e7218dd5SVikram Garhwal # define MCACHE_BUCKET_SHIFT 16
37*e7218dd5SVikram Garhwal # define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
38*e7218dd5SVikram Garhwal #else
39*e7218dd5SVikram Garhwal # define MCACHE_BUCKET_SHIFT 20
40*e7218dd5SVikram Garhwal # define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
41*e7218dd5SVikram Garhwal #endif
42*e7218dd5SVikram Garhwal #define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
43*e7218dd5SVikram Garhwal
44*e7218dd5SVikram Garhwal /* This is the size of the virtual address space reserve to QEMU that will not
45*e7218dd5SVikram Garhwal * be use by MapCache.
46*e7218dd5SVikram Garhwal * From empirical tests I observed that qemu use 75MB more than the
47*e7218dd5SVikram Garhwal * max_mcache_size.
48*e7218dd5SVikram Garhwal */
49*e7218dd5SVikram Garhwal #define NON_MCACHE_MEMORY_SIZE (80 * MiB)
50*e7218dd5SVikram Garhwal
51*e7218dd5SVikram Garhwal typedef struct MapCacheEntry {
52*e7218dd5SVikram Garhwal hwaddr paddr_index;
53*e7218dd5SVikram Garhwal uint8_t *vaddr_base;
54*e7218dd5SVikram Garhwal unsigned long *valid_mapping;
55*e7218dd5SVikram Garhwal uint32_t lock;
56*e7218dd5SVikram Garhwal #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
57*e7218dd5SVikram Garhwal uint8_t flags;
58*e7218dd5SVikram Garhwal hwaddr size;
59*e7218dd5SVikram Garhwal struct MapCacheEntry *next;
60*e7218dd5SVikram Garhwal } MapCacheEntry;
61*e7218dd5SVikram Garhwal
62*e7218dd5SVikram Garhwal typedef struct MapCacheRev {
63*e7218dd5SVikram Garhwal uint8_t *vaddr_req;
64*e7218dd5SVikram Garhwal hwaddr paddr_index;
65*e7218dd5SVikram Garhwal hwaddr size;
66*e7218dd5SVikram Garhwal QTAILQ_ENTRY(MapCacheRev) next;
67*e7218dd5SVikram Garhwal bool dma;
68*e7218dd5SVikram Garhwal } MapCacheRev;
69*e7218dd5SVikram Garhwal
70*e7218dd5SVikram Garhwal typedef struct MapCache {
71*e7218dd5SVikram Garhwal MapCacheEntry *entry;
72*e7218dd5SVikram Garhwal unsigned long nr_buckets;
73*e7218dd5SVikram Garhwal QTAILQ_HEAD(, MapCacheRev) locked_entries;
74*e7218dd5SVikram Garhwal
75*e7218dd5SVikram Garhwal /* For most cases (>99.9%), the page address is the same. */
76*e7218dd5SVikram Garhwal MapCacheEntry *last_entry;
77*e7218dd5SVikram Garhwal unsigned long max_mcache_size;
78*e7218dd5SVikram Garhwal unsigned int mcache_bucket_shift;
79*e7218dd5SVikram Garhwal
80*e7218dd5SVikram Garhwal phys_offset_to_gaddr_t phys_offset_to_gaddr;
81*e7218dd5SVikram Garhwal QemuMutex lock;
82*e7218dd5SVikram Garhwal void *opaque;
83*e7218dd5SVikram Garhwal } MapCache;
84*e7218dd5SVikram Garhwal
85*e7218dd5SVikram Garhwal static MapCache *mapcache;
86*e7218dd5SVikram Garhwal
mapcache_lock(void)87*e7218dd5SVikram Garhwal static inline void mapcache_lock(void)
88*e7218dd5SVikram Garhwal {
89*e7218dd5SVikram Garhwal qemu_mutex_lock(&mapcache->lock);
90*e7218dd5SVikram Garhwal }
91*e7218dd5SVikram Garhwal
mapcache_unlock(void)92*e7218dd5SVikram Garhwal static inline void mapcache_unlock(void)
93*e7218dd5SVikram Garhwal {
94*e7218dd5SVikram Garhwal qemu_mutex_unlock(&mapcache->lock);
95*e7218dd5SVikram Garhwal }
96*e7218dd5SVikram Garhwal
test_bits(int nr,int size,const unsigned long * addr)97*e7218dd5SVikram Garhwal static inline int test_bits(int nr, int size, const unsigned long *addr)
98*e7218dd5SVikram Garhwal {
99*e7218dd5SVikram Garhwal unsigned long res = find_next_zero_bit(addr, size + nr, nr);
100*e7218dd5SVikram Garhwal if (res >= nr + size)
101*e7218dd5SVikram Garhwal return 1;
102*e7218dd5SVikram Garhwal else
103*e7218dd5SVikram Garhwal return 0;
104*e7218dd5SVikram Garhwal }
105*e7218dd5SVikram Garhwal
xen_map_cache_init(phys_offset_to_gaddr_t f,void * opaque)106*e7218dd5SVikram Garhwal void xen_map_cache_init(phys_offset_to_gaddr_t f, void *opaque)
107*e7218dd5SVikram Garhwal {
108*e7218dd5SVikram Garhwal unsigned long size;
109*e7218dd5SVikram Garhwal struct rlimit rlimit_as;
110*e7218dd5SVikram Garhwal
111*e7218dd5SVikram Garhwal mapcache = g_new0(MapCache, 1);
112*e7218dd5SVikram Garhwal
113*e7218dd5SVikram Garhwal mapcache->phys_offset_to_gaddr = f;
114*e7218dd5SVikram Garhwal mapcache->opaque = opaque;
115*e7218dd5SVikram Garhwal qemu_mutex_init(&mapcache->lock);
116*e7218dd5SVikram Garhwal
117*e7218dd5SVikram Garhwal QTAILQ_INIT(&mapcache->locked_entries);
118*e7218dd5SVikram Garhwal
119*e7218dd5SVikram Garhwal if (geteuid() == 0) {
120*e7218dd5SVikram Garhwal rlimit_as.rlim_cur = RLIM_INFINITY;
121*e7218dd5SVikram Garhwal rlimit_as.rlim_max = RLIM_INFINITY;
122*e7218dd5SVikram Garhwal mapcache->max_mcache_size = MCACHE_MAX_SIZE;
123*e7218dd5SVikram Garhwal } else {
124*e7218dd5SVikram Garhwal getrlimit(RLIMIT_AS, &rlimit_as);
125*e7218dd5SVikram Garhwal rlimit_as.rlim_cur = rlimit_as.rlim_max;
126*e7218dd5SVikram Garhwal
127*e7218dd5SVikram Garhwal if (rlimit_as.rlim_max != RLIM_INFINITY) {
128*e7218dd5SVikram Garhwal warn_report("QEMU's maximum size of virtual"
129*e7218dd5SVikram Garhwal " memory is not infinity");
130*e7218dd5SVikram Garhwal }
131*e7218dd5SVikram Garhwal if (rlimit_as.rlim_max < MCACHE_MAX_SIZE + NON_MCACHE_MEMORY_SIZE) {
132*e7218dd5SVikram Garhwal mapcache->max_mcache_size = rlimit_as.rlim_max -
133*e7218dd5SVikram Garhwal NON_MCACHE_MEMORY_SIZE;
134*e7218dd5SVikram Garhwal } else {
135*e7218dd5SVikram Garhwal mapcache->max_mcache_size = MCACHE_MAX_SIZE;
136*e7218dd5SVikram Garhwal }
137*e7218dd5SVikram Garhwal }
138*e7218dd5SVikram Garhwal
139*e7218dd5SVikram Garhwal setrlimit(RLIMIT_AS, &rlimit_as);
140*e7218dd5SVikram Garhwal
141*e7218dd5SVikram Garhwal mapcache->nr_buckets =
142*e7218dd5SVikram Garhwal (((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
143*e7218dd5SVikram Garhwal (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
144*e7218dd5SVikram Garhwal (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
145*e7218dd5SVikram Garhwal
146*e7218dd5SVikram Garhwal size = mapcache->nr_buckets * sizeof (MapCacheEntry);
147*e7218dd5SVikram Garhwal size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
148*e7218dd5SVikram Garhwal DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
149*e7218dd5SVikram Garhwal mapcache->nr_buckets, size);
150*e7218dd5SVikram Garhwal mapcache->entry = g_malloc0(size);
151*e7218dd5SVikram Garhwal }
152*e7218dd5SVikram Garhwal
xen_remap_bucket(MapCacheEntry * entry,void * vaddr,hwaddr size,hwaddr address_index,bool dummy)153*e7218dd5SVikram Garhwal static void xen_remap_bucket(MapCacheEntry *entry,
154*e7218dd5SVikram Garhwal void *vaddr,
155*e7218dd5SVikram Garhwal hwaddr size,
156*e7218dd5SVikram Garhwal hwaddr address_index,
157*e7218dd5SVikram Garhwal bool dummy)
158*e7218dd5SVikram Garhwal {
159*e7218dd5SVikram Garhwal uint8_t *vaddr_base;
160*e7218dd5SVikram Garhwal xen_pfn_t *pfns;
161*e7218dd5SVikram Garhwal int *err;
162*e7218dd5SVikram Garhwal unsigned int i;
163*e7218dd5SVikram Garhwal hwaddr nb_pfn = size >> XC_PAGE_SHIFT;
164*e7218dd5SVikram Garhwal
165*e7218dd5SVikram Garhwal trace_xen_remap_bucket(address_index);
166*e7218dd5SVikram Garhwal
167*e7218dd5SVikram Garhwal pfns = g_new0(xen_pfn_t, nb_pfn);
168*e7218dd5SVikram Garhwal err = g_new0(int, nb_pfn);
169*e7218dd5SVikram Garhwal
170*e7218dd5SVikram Garhwal if (entry->vaddr_base != NULL) {
171*e7218dd5SVikram Garhwal if (!(entry->flags & XEN_MAPCACHE_ENTRY_DUMMY)) {
172*e7218dd5SVikram Garhwal ram_block_notify_remove(entry->vaddr_base, entry->size,
173*e7218dd5SVikram Garhwal entry->size);
174*e7218dd5SVikram Garhwal }
175*e7218dd5SVikram Garhwal
176*e7218dd5SVikram Garhwal /*
177*e7218dd5SVikram Garhwal * If an entry is being replaced by another mapping and we're using
178*e7218dd5SVikram Garhwal * MAP_FIXED flag for it - there is possibility of a race for vaddr
179*e7218dd5SVikram Garhwal * address with another thread doing an mmap call itself
180*e7218dd5SVikram Garhwal * (see man 2 mmap). To avoid that we skip explicit unmapping here
181*e7218dd5SVikram Garhwal * and allow the kernel to destroy the previous mappings by replacing
182*e7218dd5SVikram Garhwal * them in mmap call later.
183*e7218dd5SVikram Garhwal *
184*e7218dd5SVikram Garhwal * Non-identical replacements are not allowed therefore.
185*e7218dd5SVikram Garhwal */
186*e7218dd5SVikram Garhwal assert(!vaddr || (entry->vaddr_base == vaddr && entry->size == size));
187*e7218dd5SVikram Garhwal
188*e7218dd5SVikram Garhwal if (!vaddr && munmap(entry->vaddr_base, entry->size) != 0) {
189*e7218dd5SVikram Garhwal perror("unmap fails");
190*e7218dd5SVikram Garhwal exit(-1);
191*e7218dd5SVikram Garhwal }
192*e7218dd5SVikram Garhwal }
193*e7218dd5SVikram Garhwal g_free(entry->valid_mapping);
194*e7218dd5SVikram Garhwal entry->valid_mapping = NULL;
195*e7218dd5SVikram Garhwal
196*e7218dd5SVikram Garhwal for (i = 0; i < nb_pfn; i++) {
197*e7218dd5SVikram Garhwal pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
198*e7218dd5SVikram Garhwal }
199*e7218dd5SVikram Garhwal
200*e7218dd5SVikram Garhwal /*
201*e7218dd5SVikram Garhwal * If the caller has requested the mapping at a specific address use
202*e7218dd5SVikram Garhwal * MAP_FIXED to make sure it's honored.
203*e7218dd5SVikram Garhwal */
204*e7218dd5SVikram Garhwal if (!dummy) {
205*e7218dd5SVikram Garhwal vaddr_base = xenforeignmemory_map2(xen_fmem, xen_domid, vaddr,
206*e7218dd5SVikram Garhwal PROT_READ | PROT_WRITE,
207*e7218dd5SVikram Garhwal vaddr ? MAP_FIXED : 0,
208*e7218dd5SVikram Garhwal nb_pfn, pfns, err);
209*e7218dd5SVikram Garhwal if (vaddr_base == NULL) {
210*e7218dd5SVikram Garhwal perror("xenforeignmemory_map2");
211*e7218dd5SVikram Garhwal exit(-1);
212*e7218dd5SVikram Garhwal }
213*e7218dd5SVikram Garhwal } else {
214*e7218dd5SVikram Garhwal /*
215*e7218dd5SVikram Garhwal * We create dummy mappings where we are unable to create a foreign
216*e7218dd5SVikram Garhwal * mapping immediately due to certain circumstances (i.e. on resume now)
217*e7218dd5SVikram Garhwal */
218*e7218dd5SVikram Garhwal vaddr_base = mmap(vaddr, size, PROT_READ | PROT_WRITE,
219*e7218dd5SVikram Garhwal MAP_ANON | MAP_SHARED | (vaddr ? MAP_FIXED : 0),
220*e7218dd5SVikram Garhwal -1, 0);
221*e7218dd5SVikram Garhwal if (vaddr_base == MAP_FAILED) {
222*e7218dd5SVikram Garhwal perror("mmap");
223*e7218dd5SVikram Garhwal exit(-1);
224*e7218dd5SVikram Garhwal }
225*e7218dd5SVikram Garhwal }
226*e7218dd5SVikram Garhwal
227*e7218dd5SVikram Garhwal if (!(entry->flags & XEN_MAPCACHE_ENTRY_DUMMY)) {
228*e7218dd5SVikram Garhwal ram_block_notify_add(vaddr_base, size, size);
229*e7218dd5SVikram Garhwal }
230*e7218dd5SVikram Garhwal
231*e7218dd5SVikram Garhwal entry->vaddr_base = vaddr_base;
232*e7218dd5SVikram Garhwal entry->paddr_index = address_index;
233*e7218dd5SVikram Garhwal entry->size = size;
234*e7218dd5SVikram Garhwal entry->valid_mapping = g_new0(unsigned long,
235*e7218dd5SVikram Garhwal BITS_TO_LONGS(size >> XC_PAGE_SHIFT));
236*e7218dd5SVikram Garhwal
237*e7218dd5SVikram Garhwal if (dummy) {
238*e7218dd5SVikram Garhwal entry->flags |= XEN_MAPCACHE_ENTRY_DUMMY;
239*e7218dd5SVikram Garhwal } else {
240*e7218dd5SVikram Garhwal entry->flags &= ~(XEN_MAPCACHE_ENTRY_DUMMY);
241*e7218dd5SVikram Garhwal }
242*e7218dd5SVikram Garhwal
243*e7218dd5SVikram Garhwal bitmap_zero(entry->valid_mapping, nb_pfn);
244*e7218dd5SVikram Garhwal for (i = 0; i < nb_pfn; i++) {
245*e7218dd5SVikram Garhwal if (!err[i]) {
246*e7218dd5SVikram Garhwal bitmap_set(entry->valid_mapping, i, 1);
247*e7218dd5SVikram Garhwal }
248*e7218dd5SVikram Garhwal }
249*e7218dd5SVikram Garhwal
250*e7218dd5SVikram Garhwal g_free(pfns);
251*e7218dd5SVikram Garhwal g_free(err);
252*e7218dd5SVikram Garhwal }
253*e7218dd5SVikram Garhwal
xen_map_cache_unlocked(hwaddr phys_addr,hwaddr size,uint8_t lock,bool dma)254*e7218dd5SVikram Garhwal static uint8_t *xen_map_cache_unlocked(hwaddr phys_addr, hwaddr size,
255*e7218dd5SVikram Garhwal uint8_t lock, bool dma)
256*e7218dd5SVikram Garhwal {
257*e7218dd5SVikram Garhwal MapCacheEntry *entry, *pentry = NULL,
258*e7218dd5SVikram Garhwal *free_entry = NULL, *free_pentry = NULL;
259*e7218dd5SVikram Garhwal hwaddr address_index;
260*e7218dd5SVikram Garhwal hwaddr address_offset;
261*e7218dd5SVikram Garhwal hwaddr cache_size = size;
262*e7218dd5SVikram Garhwal hwaddr test_bit_size;
263*e7218dd5SVikram Garhwal bool translated G_GNUC_UNUSED = false;
264*e7218dd5SVikram Garhwal bool dummy = false;
265*e7218dd5SVikram Garhwal
266*e7218dd5SVikram Garhwal tryagain:
267*e7218dd5SVikram Garhwal address_index = phys_addr >> MCACHE_BUCKET_SHIFT;
268*e7218dd5SVikram Garhwal address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
269*e7218dd5SVikram Garhwal
270*e7218dd5SVikram Garhwal trace_xen_map_cache(phys_addr);
271*e7218dd5SVikram Garhwal
272*e7218dd5SVikram Garhwal /* test_bit_size is always a multiple of XC_PAGE_SIZE */
273*e7218dd5SVikram Garhwal if (size) {
274*e7218dd5SVikram Garhwal test_bit_size = size + (phys_addr & (XC_PAGE_SIZE - 1));
275*e7218dd5SVikram Garhwal
276*e7218dd5SVikram Garhwal if (test_bit_size % XC_PAGE_SIZE) {
277*e7218dd5SVikram Garhwal test_bit_size += XC_PAGE_SIZE - (test_bit_size % XC_PAGE_SIZE);
278*e7218dd5SVikram Garhwal }
279*e7218dd5SVikram Garhwal } else {
280*e7218dd5SVikram Garhwal test_bit_size = XC_PAGE_SIZE;
281*e7218dd5SVikram Garhwal }
282*e7218dd5SVikram Garhwal
283*e7218dd5SVikram Garhwal if (mapcache->last_entry != NULL &&
284*e7218dd5SVikram Garhwal mapcache->last_entry->paddr_index == address_index &&
285*e7218dd5SVikram Garhwal !lock && !size &&
286*e7218dd5SVikram Garhwal test_bits(address_offset >> XC_PAGE_SHIFT,
287*e7218dd5SVikram Garhwal test_bit_size >> XC_PAGE_SHIFT,
288*e7218dd5SVikram Garhwal mapcache->last_entry->valid_mapping)) {
289*e7218dd5SVikram Garhwal trace_xen_map_cache_return(mapcache->last_entry->vaddr_base + address_offset);
290*e7218dd5SVikram Garhwal return mapcache->last_entry->vaddr_base + address_offset;
291*e7218dd5SVikram Garhwal }
292*e7218dd5SVikram Garhwal
293*e7218dd5SVikram Garhwal /* size is always a multiple of MCACHE_BUCKET_SIZE */
294*e7218dd5SVikram Garhwal if (size) {
295*e7218dd5SVikram Garhwal cache_size = size + address_offset;
296*e7218dd5SVikram Garhwal if (cache_size % MCACHE_BUCKET_SIZE) {
297*e7218dd5SVikram Garhwal cache_size += MCACHE_BUCKET_SIZE - (cache_size % MCACHE_BUCKET_SIZE);
298*e7218dd5SVikram Garhwal }
299*e7218dd5SVikram Garhwal } else {
300*e7218dd5SVikram Garhwal cache_size = MCACHE_BUCKET_SIZE;
301*e7218dd5SVikram Garhwal }
302*e7218dd5SVikram Garhwal
303*e7218dd5SVikram Garhwal entry = &mapcache->entry[address_index % mapcache->nr_buckets];
304*e7218dd5SVikram Garhwal
305*e7218dd5SVikram Garhwal while (entry && (lock || entry->lock) && entry->vaddr_base &&
306*e7218dd5SVikram Garhwal (entry->paddr_index != address_index || entry->size != cache_size ||
307*e7218dd5SVikram Garhwal !test_bits(address_offset >> XC_PAGE_SHIFT,
308*e7218dd5SVikram Garhwal test_bit_size >> XC_PAGE_SHIFT,
309*e7218dd5SVikram Garhwal entry->valid_mapping))) {
310*e7218dd5SVikram Garhwal if (!free_entry && !entry->lock) {
311*e7218dd5SVikram Garhwal free_entry = entry;
312*e7218dd5SVikram Garhwal free_pentry = pentry;
313*e7218dd5SVikram Garhwal }
314*e7218dd5SVikram Garhwal pentry = entry;
315*e7218dd5SVikram Garhwal entry = entry->next;
316*e7218dd5SVikram Garhwal }
317*e7218dd5SVikram Garhwal if (!entry && free_entry) {
318*e7218dd5SVikram Garhwal entry = free_entry;
319*e7218dd5SVikram Garhwal pentry = free_pentry;
320*e7218dd5SVikram Garhwal }
321*e7218dd5SVikram Garhwal if (!entry) {
322*e7218dd5SVikram Garhwal entry = g_new0(MapCacheEntry, 1);
323*e7218dd5SVikram Garhwal pentry->next = entry;
324*e7218dd5SVikram Garhwal xen_remap_bucket(entry, NULL, cache_size, address_index, dummy);
325*e7218dd5SVikram Garhwal } else if (!entry->lock) {
326*e7218dd5SVikram Garhwal if (!entry->vaddr_base || entry->paddr_index != address_index ||
327*e7218dd5SVikram Garhwal entry->size != cache_size ||
328*e7218dd5SVikram Garhwal !test_bits(address_offset >> XC_PAGE_SHIFT,
329*e7218dd5SVikram Garhwal test_bit_size >> XC_PAGE_SHIFT,
330*e7218dd5SVikram Garhwal entry->valid_mapping)) {
331*e7218dd5SVikram Garhwal xen_remap_bucket(entry, NULL, cache_size, address_index, dummy);
332*e7218dd5SVikram Garhwal }
333*e7218dd5SVikram Garhwal }
334*e7218dd5SVikram Garhwal
335*e7218dd5SVikram Garhwal if(!test_bits(address_offset >> XC_PAGE_SHIFT,
336*e7218dd5SVikram Garhwal test_bit_size >> XC_PAGE_SHIFT,
337*e7218dd5SVikram Garhwal entry->valid_mapping)) {
338*e7218dd5SVikram Garhwal mapcache->last_entry = NULL;
339*e7218dd5SVikram Garhwal #ifdef XEN_COMPAT_PHYSMAP
340*e7218dd5SVikram Garhwal if (!translated && mapcache->phys_offset_to_gaddr) {
341*e7218dd5SVikram Garhwal phys_addr = mapcache->phys_offset_to_gaddr(phys_addr, size);
342*e7218dd5SVikram Garhwal translated = true;
343*e7218dd5SVikram Garhwal goto tryagain;
344*e7218dd5SVikram Garhwal }
345*e7218dd5SVikram Garhwal #endif
346*e7218dd5SVikram Garhwal if (!dummy && runstate_check(RUN_STATE_INMIGRATE)) {
347*e7218dd5SVikram Garhwal dummy = true;
348*e7218dd5SVikram Garhwal goto tryagain;
349*e7218dd5SVikram Garhwal }
350*e7218dd5SVikram Garhwal trace_xen_map_cache_return(NULL);
351*e7218dd5SVikram Garhwal return NULL;
352*e7218dd5SVikram Garhwal }
353*e7218dd5SVikram Garhwal
354*e7218dd5SVikram Garhwal mapcache->last_entry = entry;
355*e7218dd5SVikram Garhwal if (lock) {
356*e7218dd5SVikram Garhwal MapCacheRev *reventry = g_new0(MapCacheRev, 1);
357*e7218dd5SVikram Garhwal entry->lock++;
358*e7218dd5SVikram Garhwal if (entry->lock == 0) {
359*e7218dd5SVikram Garhwal fprintf(stderr,
360*e7218dd5SVikram Garhwal "mapcache entry lock overflow: "HWADDR_FMT_plx" -> %p\n",
361*e7218dd5SVikram Garhwal entry->paddr_index, entry->vaddr_base);
362*e7218dd5SVikram Garhwal abort();
363*e7218dd5SVikram Garhwal }
364*e7218dd5SVikram Garhwal reventry->dma = dma;
365*e7218dd5SVikram Garhwal reventry->vaddr_req = mapcache->last_entry->vaddr_base + address_offset;
366*e7218dd5SVikram Garhwal reventry->paddr_index = mapcache->last_entry->paddr_index;
367*e7218dd5SVikram Garhwal reventry->size = entry->size;
368*e7218dd5SVikram Garhwal QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
369*e7218dd5SVikram Garhwal }
370*e7218dd5SVikram Garhwal
371*e7218dd5SVikram Garhwal trace_xen_map_cache_return(mapcache->last_entry->vaddr_base + address_offset);
372*e7218dd5SVikram Garhwal return mapcache->last_entry->vaddr_base + address_offset;
373*e7218dd5SVikram Garhwal }
374*e7218dd5SVikram Garhwal
xen_map_cache(hwaddr phys_addr,hwaddr size,uint8_t lock,bool dma)375*e7218dd5SVikram Garhwal uint8_t *xen_map_cache(hwaddr phys_addr, hwaddr size,
376*e7218dd5SVikram Garhwal uint8_t lock, bool dma)
377*e7218dd5SVikram Garhwal {
378*e7218dd5SVikram Garhwal uint8_t *p;
379*e7218dd5SVikram Garhwal
380*e7218dd5SVikram Garhwal mapcache_lock();
381*e7218dd5SVikram Garhwal p = xen_map_cache_unlocked(phys_addr, size, lock, dma);
382*e7218dd5SVikram Garhwal mapcache_unlock();
383*e7218dd5SVikram Garhwal return p;
384*e7218dd5SVikram Garhwal }
385*e7218dd5SVikram Garhwal
xen_ram_addr_from_mapcache(void * ptr)386*e7218dd5SVikram Garhwal ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
387*e7218dd5SVikram Garhwal {
388*e7218dd5SVikram Garhwal MapCacheEntry *entry = NULL;
389*e7218dd5SVikram Garhwal MapCacheRev *reventry;
390*e7218dd5SVikram Garhwal hwaddr paddr_index;
391*e7218dd5SVikram Garhwal hwaddr size;
392*e7218dd5SVikram Garhwal ram_addr_t raddr;
393*e7218dd5SVikram Garhwal int found = 0;
394*e7218dd5SVikram Garhwal
395*e7218dd5SVikram Garhwal mapcache_lock();
396*e7218dd5SVikram Garhwal QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
397*e7218dd5SVikram Garhwal if (reventry->vaddr_req == ptr) {
398*e7218dd5SVikram Garhwal paddr_index = reventry->paddr_index;
399*e7218dd5SVikram Garhwal size = reventry->size;
400*e7218dd5SVikram Garhwal found = 1;
401*e7218dd5SVikram Garhwal break;
402*e7218dd5SVikram Garhwal }
403*e7218dd5SVikram Garhwal }
404*e7218dd5SVikram Garhwal if (!found) {
405*e7218dd5SVikram Garhwal fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
406*e7218dd5SVikram Garhwal QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
407*e7218dd5SVikram Garhwal DPRINTF(" "HWADDR_FMT_plx" -> %p is present\n", reventry->paddr_index,
408*e7218dd5SVikram Garhwal reventry->vaddr_req);
409*e7218dd5SVikram Garhwal }
410*e7218dd5SVikram Garhwal abort();
411*e7218dd5SVikram Garhwal return 0;
412*e7218dd5SVikram Garhwal }
413*e7218dd5SVikram Garhwal
414*e7218dd5SVikram Garhwal entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
415*e7218dd5SVikram Garhwal while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
416*e7218dd5SVikram Garhwal entry = entry->next;
417*e7218dd5SVikram Garhwal }
418*e7218dd5SVikram Garhwal if (!entry) {
419*e7218dd5SVikram Garhwal DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
420*e7218dd5SVikram Garhwal raddr = 0;
421*e7218dd5SVikram Garhwal } else {
422*e7218dd5SVikram Garhwal raddr = (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
423*e7218dd5SVikram Garhwal ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
424*e7218dd5SVikram Garhwal }
425*e7218dd5SVikram Garhwal mapcache_unlock();
426*e7218dd5SVikram Garhwal return raddr;
427*e7218dd5SVikram Garhwal }
428*e7218dd5SVikram Garhwal
xen_invalidate_map_cache_entry_unlocked(uint8_t * buffer)429*e7218dd5SVikram Garhwal static void xen_invalidate_map_cache_entry_unlocked(uint8_t *buffer)
430*e7218dd5SVikram Garhwal {
431*e7218dd5SVikram Garhwal MapCacheEntry *entry = NULL, *pentry = NULL;
432*e7218dd5SVikram Garhwal MapCacheRev *reventry;
433*e7218dd5SVikram Garhwal hwaddr paddr_index;
434*e7218dd5SVikram Garhwal hwaddr size;
435*e7218dd5SVikram Garhwal int found = 0;
436*e7218dd5SVikram Garhwal
437*e7218dd5SVikram Garhwal QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
438*e7218dd5SVikram Garhwal if (reventry->vaddr_req == buffer) {
439*e7218dd5SVikram Garhwal paddr_index = reventry->paddr_index;
440*e7218dd5SVikram Garhwal size = reventry->size;
441*e7218dd5SVikram Garhwal found = 1;
442*e7218dd5SVikram Garhwal break;
443*e7218dd5SVikram Garhwal }
444*e7218dd5SVikram Garhwal }
445*e7218dd5SVikram Garhwal if (!found) {
446*e7218dd5SVikram Garhwal DPRINTF("%s, could not find %p\n", __func__, buffer);
447*e7218dd5SVikram Garhwal QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
448*e7218dd5SVikram Garhwal DPRINTF(" "HWADDR_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
449*e7218dd5SVikram Garhwal }
450*e7218dd5SVikram Garhwal return;
451*e7218dd5SVikram Garhwal }
452*e7218dd5SVikram Garhwal QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
453*e7218dd5SVikram Garhwal g_free(reventry);
454*e7218dd5SVikram Garhwal
455*e7218dd5SVikram Garhwal if (mapcache->last_entry != NULL &&
456*e7218dd5SVikram Garhwal mapcache->last_entry->paddr_index == paddr_index) {
457*e7218dd5SVikram Garhwal mapcache->last_entry = NULL;
458*e7218dd5SVikram Garhwal }
459*e7218dd5SVikram Garhwal
460*e7218dd5SVikram Garhwal entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
461*e7218dd5SVikram Garhwal while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
462*e7218dd5SVikram Garhwal pentry = entry;
463*e7218dd5SVikram Garhwal entry = entry->next;
464*e7218dd5SVikram Garhwal }
465*e7218dd5SVikram Garhwal if (!entry) {
466*e7218dd5SVikram Garhwal DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
467*e7218dd5SVikram Garhwal return;
468*e7218dd5SVikram Garhwal }
469*e7218dd5SVikram Garhwal entry->lock--;
470*e7218dd5SVikram Garhwal if (entry->lock > 0 || pentry == NULL) {
471*e7218dd5SVikram Garhwal return;
472*e7218dd5SVikram Garhwal }
473*e7218dd5SVikram Garhwal
474*e7218dd5SVikram Garhwal pentry->next = entry->next;
475*e7218dd5SVikram Garhwal ram_block_notify_remove(entry->vaddr_base, entry->size, entry->size);
476*e7218dd5SVikram Garhwal if (munmap(entry->vaddr_base, entry->size) != 0) {
477*e7218dd5SVikram Garhwal perror("unmap fails");
478*e7218dd5SVikram Garhwal exit(-1);
479*e7218dd5SVikram Garhwal }
480*e7218dd5SVikram Garhwal g_free(entry->valid_mapping);
481*e7218dd5SVikram Garhwal g_free(entry);
482*e7218dd5SVikram Garhwal }
483*e7218dd5SVikram Garhwal
xen_invalidate_map_cache_entry(uint8_t * buffer)484*e7218dd5SVikram Garhwal void xen_invalidate_map_cache_entry(uint8_t *buffer)
485*e7218dd5SVikram Garhwal {
486*e7218dd5SVikram Garhwal mapcache_lock();
487*e7218dd5SVikram Garhwal xen_invalidate_map_cache_entry_unlocked(buffer);
488*e7218dd5SVikram Garhwal mapcache_unlock();
489*e7218dd5SVikram Garhwal }
490*e7218dd5SVikram Garhwal
xen_invalidate_map_cache(void)491*e7218dd5SVikram Garhwal void xen_invalidate_map_cache(void)
492*e7218dd5SVikram Garhwal {
493*e7218dd5SVikram Garhwal unsigned long i;
494*e7218dd5SVikram Garhwal MapCacheRev *reventry;
495*e7218dd5SVikram Garhwal
496*e7218dd5SVikram Garhwal /* Flush pending AIO before destroying the mapcache */
497*e7218dd5SVikram Garhwal bdrv_drain_all();
498*e7218dd5SVikram Garhwal
499*e7218dd5SVikram Garhwal mapcache_lock();
500*e7218dd5SVikram Garhwal
501*e7218dd5SVikram Garhwal QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
502*e7218dd5SVikram Garhwal if (!reventry->dma) {
503*e7218dd5SVikram Garhwal continue;
504*e7218dd5SVikram Garhwal }
505*e7218dd5SVikram Garhwal fprintf(stderr, "Locked DMA mapping while invalidating mapcache!"
506*e7218dd5SVikram Garhwal " "HWADDR_FMT_plx" -> %p is present\n",
507*e7218dd5SVikram Garhwal reventry->paddr_index, reventry->vaddr_req);
508*e7218dd5SVikram Garhwal }
509*e7218dd5SVikram Garhwal
510*e7218dd5SVikram Garhwal for (i = 0; i < mapcache->nr_buckets; i++) {
511*e7218dd5SVikram Garhwal MapCacheEntry *entry = &mapcache->entry[i];
512*e7218dd5SVikram Garhwal
513*e7218dd5SVikram Garhwal if (entry->vaddr_base == NULL) {
514*e7218dd5SVikram Garhwal continue;
515*e7218dd5SVikram Garhwal }
516*e7218dd5SVikram Garhwal if (entry->lock > 0) {
517*e7218dd5SVikram Garhwal continue;
518*e7218dd5SVikram Garhwal }
519*e7218dd5SVikram Garhwal
520*e7218dd5SVikram Garhwal if (munmap(entry->vaddr_base, entry->size) != 0) {
521*e7218dd5SVikram Garhwal perror("unmap fails");
522*e7218dd5SVikram Garhwal exit(-1);
523*e7218dd5SVikram Garhwal }
524*e7218dd5SVikram Garhwal
525*e7218dd5SVikram Garhwal entry->paddr_index = 0;
526*e7218dd5SVikram Garhwal entry->vaddr_base = NULL;
527*e7218dd5SVikram Garhwal entry->size = 0;
528*e7218dd5SVikram Garhwal g_free(entry->valid_mapping);
529*e7218dd5SVikram Garhwal entry->valid_mapping = NULL;
530*e7218dd5SVikram Garhwal }
531*e7218dd5SVikram Garhwal
532*e7218dd5SVikram Garhwal mapcache->last_entry = NULL;
533*e7218dd5SVikram Garhwal
534*e7218dd5SVikram Garhwal mapcache_unlock();
535*e7218dd5SVikram Garhwal }
536*e7218dd5SVikram Garhwal
xen_replace_cache_entry_unlocked(hwaddr old_phys_addr,hwaddr new_phys_addr,hwaddr size)537*e7218dd5SVikram Garhwal static uint8_t *xen_replace_cache_entry_unlocked(hwaddr old_phys_addr,
538*e7218dd5SVikram Garhwal hwaddr new_phys_addr,
539*e7218dd5SVikram Garhwal hwaddr size)
540*e7218dd5SVikram Garhwal {
541*e7218dd5SVikram Garhwal MapCacheEntry *entry;
542*e7218dd5SVikram Garhwal hwaddr address_index, address_offset;
543*e7218dd5SVikram Garhwal hwaddr test_bit_size, cache_size = size;
544*e7218dd5SVikram Garhwal
545*e7218dd5SVikram Garhwal address_index = old_phys_addr >> MCACHE_BUCKET_SHIFT;
546*e7218dd5SVikram Garhwal address_offset = old_phys_addr & (MCACHE_BUCKET_SIZE - 1);
547*e7218dd5SVikram Garhwal
548*e7218dd5SVikram Garhwal assert(size);
549*e7218dd5SVikram Garhwal /* test_bit_size is always a multiple of XC_PAGE_SIZE */
550*e7218dd5SVikram Garhwal test_bit_size = size + (old_phys_addr & (XC_PAGE_SIZE - 1));
551*e7218dd5SVikram Garhwal if (test_bit_size % XC_PAGE_SIZE) {
552*e7218dd5SVikram Garhwal test_bit_size += XC_PAGE_SIZE - (test_bit_size % XC_PAGE_SIZE);
553*e7218dd5SVikram Garhwal }
554*e7218dd5SVikram Garhwal cache_size = size + address_offset;
555*e7218dd5SVikram Garhwal if (cache_size % MCACHE_BUCKET_SIZE) {
556*e7218dd5SVikram Garhwal cache_size += MCACHE_BUCKET_SIZE - (cache_size % MCACHE_BUCKET_SIZE);
557*e7218dd5SVikram Garhwal }
558*e7218dd5SVikram Garhwal
559*e7218dd5SVikram Garhwal entry = &mapcache->entry[address_index % mapcache->nr_buckets];
560*e7218dd5SVikram Garhwal while (entry && !(entry->paddr_index == address_index &&
561*e7218dd5SVikram Garhwal entry->size == cache_size)) {
562*e7218dd5SVikram Garhwal entry = entry->next;
563*e7218dd5SVikram Garhwal }
564*e7218dd5SVikram Garhwal if (!entry) {
565*e7218dd5SVikram Garhwal DPRINTF("Trying to update an entry for "HWADDR_FMT_plx \
566*e7218dd5SVikram Garhwal "that is not in the mapcache!\n", old_phys_addr);
567*e7218dd5SVikram Garhwal return NULL;
568*e7218dd5SVikram Garhwal }
569*e7218dd5SVikram Garhwal
570*e7218dd5SVikram Garhwal address_index = new_phys_addr >> MCACHE_BUCKET_SHIFT;
571*e7218dd5SVikram Garhwal address_offset = new_phys_addr & (MCACHE_BUCKET_SIZE - 1);
572*e7218dd5SVikram Garhwal
573*e7218dd5SVikram Garhwal fprintf(stderr, "Replacing a dummy mapcache entry for "HWADDR_FMT_plx \
574*e7218dd5SVikram Garhwal " with "HWADDR_FMT_plx"\n", old_phys_addr, new_phys_addr);
575*e7218dd5SVikram Garhwal
576*e7218dd5SVikram Garhwal xen_remap_bucket(entry, entry->vaddr_base,
577*e7218dd5SVikram Garhwal cache_size, address_index, false);
578*e7218dd5SVikram Garhwal if (!test_bits(address_offset >> XC_PAGE_SHIFT,
579*e7218dd5SVikram Garhwal test_bit_size >> XC_PAGE_SHIFT,
580*e7218dd5SVikram Garhwal entry->valid_mapping)) {
581*e7218dd5SVikram Garhwal DPRINTF("Unable to update a mapcache entry for "HWADDR_FMT_plx"!\n",
582*e7218dd5SVikram Garhwal old_phys_addr);
583*e7218dd5SVikram Garhwal return NULL;
584*e7218dd5SVikram Garhwal }
585*e7218dd5SVikram Garhwal
586*e7218dd5SVikram Garhwal return entry->vaddr_base + address_offset;
587*e7218dd5SVikram Garhwal }
588*e7218dd5SVikram Garhwal
xen_replace_cache_entry(hwaddr old_phys_addr,hwaddr new_phys_addr,hwaddr size)589*e7218dd5SVikram Garhwal uint8_t *xen_replace_cache_entry(hwaddr old_phys_addr,
590*e7218dd5SVikram Garhwal hwaddr new_phys_addr,
591*e7218dd5SVikram Garhwal hwaddr size)
592*e7218dd5SVikram Garhwal {
593*e7218dd5SVikram Garhwal uint8_t *p;
594*e7218dd5SVikram Garhwal
595*e7218dd5SVikram Garhwal mapcache_lock();
596*e7218dd5SVikram Garhwal p = xen_replace_cache_entry_unlocked(old_phys_addr, new_phys_addr, size);
597*e7218dd5SVikram Garhwal mapcache_unlock();
598*e7218dd5SVikram Garhwal return p;
599*e7218dd5SVikram Garhwal }
600