1b7468b15SThomas Hellstrom // SPDX-License-Identifier: GPL-2.0 OR MIT
2b7468b15SThomas Hellstrom /**************************************************************************
3b7468b15SThomas Hellstrom *
409881d29SZack Rusin * Copyright 2019-2023 VMware, Inc., Palo Alto, CA., USA
5b7468b15SThomas Hellstrom *
6b7468b15SThomas Hellstrom * Permission is hereby granted, free of charge, to any person obtaining a
7b7468b15SThomas Hellstrom * copy of this software and associated documentation files (the
8b7468b15SThomas Hellstrom * "Software"), to deal in the Software without restriction, including
9b7468b15SThomas Hellstrom * without limitation the rights to use, copy, modify, merge, publish,
10b7468b15SThomas Hellstrom * distribute, sub license, and/or sell copies of the Software, and to
11b7468b15SThomas Hellstrom * permit persons to whom the Software is furnished to do so, subject to
12b7468b15SThomas Hellstrom * the following conditions:
13b7468b15SThomas Hellstrom *
14b7468b15SThomas Hellstrom * The above copyright notice and this permission notice (including the
15b7468b15SThomas Hellstrom * next paragraph) shall be included in all copies or substantial portions
16b7468b15SThomas Hellstrom * of the Software.
17b7468b15SThomas Hellstrom *
18b7468b15SThomas Hellstrom * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19b7468b15SThomas Hellstrom * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20b7468b15SThomas Hellstrom * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21b7468b15SThomas Hellstrom * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22b7468b15SThomas Hellstrom * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23b7468b15SThomas Hellstrom * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24b7468b15SThomas Hellstrom * USE OR OTHER DEALINGS IN THE SOFTWARE.
25b7468b15SThomas Hellstrom *
26b7468b15SThomas Hellstrom **************************************************************************/
2709881d29SZack Rusin #include "vmwgfx_bo.h"
28b7468b15SThomas Hellstrom #include "vmwgfx_drv.h"
29b7468b15SThomas Hellstrom
30b7468b15SThomas Hellstrom /*
31b7468b15SThomas Hellstrom * Different methods for tracking dirty:
32b7468b15SThomas Hellstrom * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits
33b7468b15SThomas Hellstrom * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write-
34b7468b15SThomas Hellstrom * accesses in the VM mkwrite() callback
35b7468b15SThomas Hellstrom */
36b7468b15SThomas Hellstrom enum vmw_bo_dirty_method {
37b7468b15SThomas Hellstrom VMW_BO_DIRTY_PAGETABLE,
38b7468b15SThomas Hellstrom VMW_BO_DIRTY_MKWRITE,
39b7468b15SThomas Hellstrom };
40b7468b15SThomas Hellstrom
41b7468b15SThomas Hellstrom /*
42b7468b15SThomas Hellstrom * No dirtied pages at scan trigger a transition to the _MKWRITE method,
43b7468b15SThomas Hellstrom * similarly a certain percentage of dirty pages trigger a transition to
44b7468b15SThomas Hellstrom * the _PAGETABLE method. How many triggers should we wait for before
45b7468b15SThomas Hellstrom * changing method?
46b7468b15SThomas Hellstrom */
47b7468b15SThomas Hellstrom #define VMW_DIRTY_NUM_CHANGE_TRIGGERS 2
48b7468b15SThomas Hellstrom
49b7468b15SThomas Hellstrom /* Percentage to trigger a transition to the _PAGETABLE method */
50b7468b15SThomas Hellstrom #define VMW_DIRTY_PERCENTAGE 10
51b7468b15SThomas Hellstrom
52b7468b15SThomas Hellstrom /**
53b7468b15SThomas Hellstrom * struct vmw_bo_dirty - Dirty information for buffer objects
54b7468b15SThomas Hellstrom * @start: First currently dirty bit
55b7468b15SThomas Hellstrom * @end: Last currently dirty bit + 1
56b7468b15SThomas Hellstrom * @method: The currently used dirty method
57b7468b15SThomas Hellstrom * @change_count: Number of consecutive method change triggers
58b7468b15SThomas Hellstrom * @ref_count: Reference count for this structure
59b7468b15SThomas Hellstrom * @bitmap_size: The size of the bitmap in bits. Typically equal to the
60b7468b15SThomas Hellstrom * nuber of pages in the bo.
61b7468b15SThomas Hellstrom * @bitmap: A bitmap where each bit represents a page. A set bit means a
62b7468b15SThomas Hellstrom * dirty page.
63b7468b15SThomas Hellstrom */
64b7468b15SThomas Hellstrom struct vmw_bo_dirty {
65b7468b15SThomas Hellstrom unsigned long start;
66b7468b15SThomas Hellstrom unsigned long end;
67b7468b15SThomas Hellstrom enum vmw_bo_dirty_method method;
68b7468b15SThomas Hellstrom unsigned int change_count;
69b7468b15SThomas Hellstrom unsigned int ref_count;
70b7468b15SThomas Hellstrom unsigned long bitmap_size;
716b656755SGustavo A. R. Silva unsigned long bitmap[];
72b7468b15SThomas Hellstrom };
73b7468b15SThomas Hellstrom
74b7468b15SThomas Hellstrom /**
75b7468b15SThomas Hellstrom * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits
76b7468b15SThomas Hellstrom * @vbo: The buffer object to scan
77b7468b15SThomas Hellstrom *
78b7468b15SThomas Hellstrom * Scans the pagetable for dirty bits. Clear those bits and modify the
79b7468b15SThomas Hellstrom * dirty structure with the results. This function may change the
80b7468b15SThomas Hellstrom * dirty-tracking method.
81b7468b15SThomas Hellstrom */
vmw_bo_dirty_scan_pagetable(struct vmw_bo * vbo)8209881d29SZack Rusin static void vmw_bo_dirty_scan_pagetable(struct vmw_bo *vbo)
83b7468b15SThomas Hellstrom {
84b7468b15SThomas Hellstrom struct vmw_bo_dirty *dirty = vbo->dirty;
85*668b2066SZack Rusin pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
86*668b2066SZack Rusin struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
87b7468b15SThomas Hellstrom pgoff_t num_marked;
88b7468b15SThomas Hellstrom
89b7468b15SThomas Hellstrom num_marked = clean_record_shared_mapping_range
90b7468b15SThomas Hellstrom (mapping,
91b7468b15SThomas Hellstrom offset, dirty->bitmap_size,
92b7468b15SThomas Hellstrom offset, &dirty->bitmap[0],
93b7468b15SThomas Hellstrom &dirty->start, &dirty->end);
94b7468b15SThomas Hellstrom if (num_marked == 0)
95b7468b15SThomas Hellstrom dirty->change_count++;
96b7468b15SThomas Hellstrom else
97b7468b15SThomas Hellstrom dirty->change_count = 0;
98b7468b15SThomas Hellstrom
99b7468b15SThomas Hellstrom if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
100b7468b15SThomas Hellstrom dirty->change_count = 0;
101b7468b15SThomas Hellstrom dirty->method = VMW_BO_DIRTY_MKWRITE;
102b7468b15SThomas Hellstrom wp_shared_mapping_range(mapping,
103b7468b15SThomas Hellstrom offset, dirty->bitmap_size);
104b7468b15SThomas Hellstrom clean_record_shared_mapping_range(mapping,
105b7468b15SThomas Hellstrom offset, dirty->bitmap_size,
106b7468b15SThomas Hellstrom offset, &dirty->bitmap[0],
107b7468b15SThomas Hellstrom &dirty->start, &dirty->end);
108b7468b15SThomas Hellstrom }
109b7468b15SThomas Hellstrom }
110b7468b15SThomas Hellstrom
111b7468b15SThomas Hellstrom /**
112b7468b15SThomas Hellstrom * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method
113b7468b15SThomas Hellstrom * @vbo: The buffer object to scan
114b7468b15SThomas Hellstrom *
115b7468b15SThomas Hellstrom * Write-protect pages written to so that consecutive write accesses will
116b7468b15SThomas Hellstrom * trigger a call to mkwrite.
117b7468b15SThomas Hellstrom *
118b7468b15SThomas Hellstrom * This function may change the dirty-tracking method.
119b7468b15SThomas Hellstrom */
vmw_bo_dirty_scan_mkwrite(struct vmw_bo * vbo)12009881d29SZack Rusin static void vmw_bo_dirty_scan_mkwrite(struct vmw_bo *vbo)
121b7468b15SThomas Hellstrom {
122b7468b15SThomas Hellstrom struct vmw_bo_dirty *dirty = vbo->dirty;
123*668b2066SZack Rusin unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
124*668b2066SZack Rusin struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
125b7468b15SThomas Hellstrom pgoff_t num_marked;
126b7468b15SThomas Hellstrom
127b7468b15SThomas Hellstrom if (dirty->end <= dirty->start)
128b7468b15SThomas Hellstrom return;
129b7468b15SThomas Hellstrom
130*668b2066SZack Rusin num_marked = wp_shared_mapping_range(vbo->tbo.bdev->dev_mapping,
131b7468b15SThomas Hellstrom dirty->start + offset,
132b7468b15SThomas Hellstrom dirty->end - dirty->start);
133b7468b15SThomas Hellstrom
134b7468b15SThomas Hellstrom if (100UL * num_marked / dirty->bitmap_size >
135*668b2066SZack Rusin VMW_DIRTY_PERCENTAGE)
136b7468b15SThomas Hellstrom dirty->change_count++;
137*668b2066SZack Rusin else
138b7468b15SThomas Hellstrom dirty->change_count = 0;
139b7468b15SThomas Hellstrom
140b7468b15SThomas Hellstrom if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
141b7468b15SThomas Hellstrom pgoff_t start = 0;
142b7468b15SThomas Hellstrom pgoff_t end = dirty->bitmap_size;
143b7468b15SThomas Hellstrom
144b7468b15SThomas Hellstrom dirty->method = VMW_BO_DIRTY_PAGETABLE;
145b7468b15SThomas Hellstrom clean_record_shared_mapping_range(mapping, offset, end, offset,
146b7468b15SThomas Hellstrom &dirty->bitmap[0],
147b7468b15SThomas Hellstrom &start, &end);
148b7468b15SThomas Hellstrom bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size);
149b7468b15SThomas Hellstrom if (dirty->start < dirty->end)
150b7468b15SThomas Hellstrom bitmap_set(&dirty->bitmap[0], dirty->start,
151b7468b15SThomas Hellstrom dirty->end - dirty->start);
152b7468b15SThomas Hellstrom dirty->change_count = 0;
153b7468b15SThomas Hellstrom }
154b7468b15SThomas Hellstrom }
155b7468b15SThomas Hellstrom
156b7468b15SThomas Hellstrom /**
157b7468b15SThomas Hellstrom * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty
158b7468b15SThomas Hellstrom * tracking structure
159b7468b15SThomas Hellstrom * @vbo: The buffer object to scan
160b7468b15SThomas Hellstrom *
161b7468b15SThomas Hellstrom * This function may change the dirty tracking method.
162b7468b15SThomas Hellstrom */
vmw_bo_dirty_scan(struct vmw_bo * vbo)16309881d29SZack Rusin void vmw_bo_dirty_scan(struct vmw_bo *vbo)
164b7468b15SThomas Hellstrom {
165b7468b15SThomas Hellstrom struct vmw_bo_dirty *dirty = vbo->dirty;
166b7468b15SThomas Hellstrom
167b7468b15SThomas Hellstrom if (dirty->method == VMW_BO_DIRTY_PAGETABLE)
168b7468b15SThomas Hellstrom vmw_bo_dirty_scan_pagetable(vbo);
169b7468b15SThomas Hellstrom else
170b7468b15SThomas Hellstrom vmw_bo_dirty_scan_mkwrite(vbo);
171b7468b15SThomas Hellstrom }
172b7468b15SThomas Hellstrom
173b7468b15SThomas Hellstrom /**
174fb80edb0SThomas Hellstrom * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before
175fb80edb0SThomas Hellstrom * an unmap_mapping_range operation.
176fb80edb0SThomas Hellstrom * @vbo: The buffer object,
177fb80edb0SThomas Hellstrom * @start: First page of the range within the buffer object.
178fb80edb0SThomas Hellstrom * @end: Last page of the range within the buffer object + 1.
179fb80edb0SThomas Hellstrom *
180fb80edb0SThomas Hellstrom * If we're using the _PAGETABLE scan method, we may leak dirty pages
181fb80edb0SThomas Hellstrom * when calling unmap_mapping_range(). This function makes sure we pick
182fb80edb0SThomas Hellstrom * up all dirty pages.
183fb80edb0SThomas Hellstrom */
vmw_bo_dirty_pre_unmap(struct vmw_bo * vbo,pgoff_t start,pgoff_t end)18409881d29SZack Rusin static void vmw_bo_dirty_pre_unmap(struct vmw_bo *vbo,
185fb80edb0SThomas Hellstrom pgoff_t start, pgoff_t end)
186fb80edb0SThomas Hellstrom {
187fb80edb0SThomas Hellstrom struct vmw_bo_dirty *dirty = vbo->dirty;
188*668b2066SZack Rusin unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
189*668b2066SZack Rusin struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
190fb80edb0SThomas Hellstrom
191fb80edb0SThomas Hellstrom if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end)
192fb80edb0SThomas Hellstrom return;
193fb80edb0SThomas Hellstrom
194fb80edb0SThomas Hellstrom wp_shared_mapping_range(mapping, start + offset, end - start);
195fb80edb0SThomas Hellstrom clean_record_shared_mapping_range(mapping, start + offset,
196fb80edb0SThomas Hellstrom end - start, offset,
197fb80edb0SThomas Hellstrom &dirty->bitmap[0], &dirty->start,
198fb80edb0SThomas Hellstrom &dirty->end);
199fb80edb0SThomas Hellstrom }
200fb80edb0SThomas Hellstrom
201fb80edb0SThomas Hellstrom /**
202fb80edb0SThomas Hellstrom * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo
203fb80edb0SThomas Hellstrom * @vbo: The buffer object,
204fb80edb0SThomas Hellstrom * @start: First page of the range within the buffer object.
205fb80edb0SThomas Hellstrom * @end: Last page of the range within the buffer object + 1.
206fb80edb0SThomas Hellstrom *
20772dc6e3bSChristian König * This is similar to ttm_bo_unmap_virtual() except it takes a subrange.
208fb80edb0SThomas Hellstrom */
vmw_bo_dirty_unmap(struct vmw_bo * vbo,pgoff_t start,pgoff_t end)20909881d29SZack Rusin void vmw_bo_dirty_unmap(struct vmw_bo *vbo,
210fb80edb0SThomas Hellstrom pgoff_t start, pgoff_t end)
211fb80edb0SThomas Hellstrom {
212*668b2066SZack Rusin unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
213*668b2066SZack Rusin struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
214fb80edb0SThomas Hellstrom
215fb80edb0SThomas Hellstrom vmw_bo_dirty_pre_unmap(vbo, start, end);
216fb80edb0SThomas Hellstrom unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT,
217fb80edb0SThomas Hellstrom (loff_t) (end - start) << PAGE_SHIFT);
218fb80edb0SThomas Hellstrom }
219fb80edb0SThomas Hellstrom
220fb80edb0SThomas Hellstrom /**
221b7468b15SThomas Hellstrom * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object
222b7468b15SThomas Hellstrom * @vbo: The buffer object
223b7468b15SThomas Hellstrom *
224b7468b15SThomas Hellstrom * This function registers a dirty-tracking user to a buffer object.
225b7468b15SThomas Hellstrom * A user can be for example a resource or a vma in a special user-space
226b7468b15SThomas Hellstrom * mapping.
227b7468b15SThomas Hellstrom *
228b7468b15SThomas Hellstrom * Return: Zero on success, -ENOMEM on memory allocation failure.
229b7468b15SThomas Hellstrom */
vmw_bo_dirty_add(struct vmw_bo * vbo)23009881d29SZack Rusin int vmw_bo_dirty_add(struct vmw_bo *vbo)
231b7468b15SThomas Hellstrom {
232b7468b15SThomas Hellstrom struct vmw_bo_dirty *dirty = vbo->dirty;
233*668b2066SZack Rusin pgoff_t num_pages = PFN_UP(vbo->tbo.resource->size);
2348aadeb8aSZack Rusin size_t size;
235b7468b15SThomas Hellstrom int ret;
236b7468b15SThomas Hellstrom
237b7468b15SThomas Hellstrom if (dirty) {
238b7468b15SThomas Hellstrom dirty->ref_count++;
239b7468b15SThomas Hellstrom return 0;
240b7468b15SThomas Hellstrom }
241b7468b15SThomas Hellstrom
242b7468b15SThomas Hellstrom size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long);
243b7468b15SThomas Hellstrom dirty = kvzalloc(size, GFP_KERNEL);
244b7468b15SThomas Hellstrom if (!dirty) {
245b7468b15SThomas Hellstrom ret = -ENOMEM;
246b7468b15SThomas Hellstrom goto out_no_dirty;
247b7468b15SThomas Hellstrom }
248b7468b15SThomas Hellstrom
249b7468b15SThomas Hellstrom dirty->bitmap_size = num_pages;
250b7468b15SThomas Hellstrom dirty->start = dirty->bitmap_size;
251b7468b15SThomas Hellstrom dirty->end = 0;
252b7468b15SThomas Hellstrom dirty->ref_count = 1;
253b7468b15SThomas Hellstrom if (num_pages < PAGE_SIZE / sizeof(pte_t)) {
254b7468b15SThomas Hellstrom dirty->method = VMW_BO_DIRTY_PAGETABLE;
255b7468b15SThomas Hellstrom } else {
256*668b2066SZack Rusin struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
257*668b2066SZack Rusin pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
258b7468b15SThomas Hellstrom
259b7468b15SThomas Hellstrom dirty->method = VMW_BO_DIRTY_MKWRITE;
260b7468b15SThomas Hellstrom
261b7468b15SThomas Hellstrom /* Write-protect and then pick up already dirty bits */
262b7468b15SThomas Hellstrom wp_shared_mapping_range(mapping, offset, num_pages);
263b7468b15SThomas Hellstrom clean_record_shared_mapping_range(mapping, offset, num_pages,
264b7468b15SThomas Hellstrom offset,
265b7468b15SThomas Hellstrom &dirty->bitmap[0],
266b7468b15SThomas Hellstrom &dirty->start, &dirty->end);
267b7468b15SThomas Hellstrom }
268b7468b15SThomas Hellstrom
269b7468b15SThomas Hellstrom vbo->dirty = dirty;
270b7468b15SThomas Hellstrom
271b7468b15SThomas Hellstrom return 0;
272b7468b15SThomas Hellstrom
273b7468b15SThomas Hellstrom out_no_dirty:
274b7468b15SThomas Hellstrom return ret;
275b7468b15SThomas Hellstrom }
276b7468b15SThomas Hellstrom
277b7468b15SThomas Hellstrom /**
278b7468b15SThomas Hellstrom * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object
279b7468b15SThomas Hellstrom * @vbo: The buffer object
280b7468b15SThomas Hellstrom *
281b7468b15SThomas Hellstrom * This function releases a dirty-tracking user from a buffer object.
282b7468b15SThomas Hellstrom * If the reference count reaches zero, then the dirty-tracking object is
283b7468b15SThomas Hellstrom * freed and the pointer to it cleared.
284b7468b15SThomas Hellstrom *
285b7468b15SThomas Hellstrom * Return: Zero on success, -ENOMEM on memory allocation failure.
286b7468b15SThomas Hellstrom */
vmw_bo_dirty_release(struct vmw_bo * vbo)28709881d29SZack Rusin void vmw_bo_dirty_release(struct vmw_bo *vbo)
288b7468b15SThomas Hellstrom {
289b7468b15SThomas Hellstrom struct vmw_bo_dirty *dirty = vbo->dirty;
290b7468b15SThomas Hellstrom
291b7468b15SThomas Hellstrom if (dirty && --dirty->ref_count == 0) {
292b7468b15SThomas Hellstrom kvfree(dirty);
293b7468b15SThomas Hellstrom vbo->dirty = NULL;
294b7468b15SThomas Hellstrom }
295b7468b15SThomas Hellstrom }
296b7468b15SThomas Hellstrom
297b7468b15SThomas Hellstrom /**
298b7468b15SThomas Hellstrom * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from
299b7468b15SThomas Hellstrom * its backing mob.
300b7468b15SThomas Hellstrom * @res: The resource
301b7468b15SThomas Hellstrom *
302b7468b15SThomas Hellstrom * This function will pick up all dirty ranges affecting the resource from
303b7468b15SThomas Hellstrom * it's backup mob, and call vmw_resource_dirty_update() once for each
304b7468b15SThomas Hellstrom * range. The transferred ranges will be cleared from the backing mob's
305b7468b15SThomas Hellstrom * dirty tracking.
306b7468b15SThomas Hellstrom */
vmw_bo_dirty_transfer_to_res(struct vmw_resource * res)307b7468b15SThomas Hellstrom void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)
308b7468b15SThomas Hellstrom {
309*668b2066SZack Rusin struct vmw_bo *vbo = res->guest_memory_bo;
310b7468b15SThomas Hellstrom struct vmw_bo_dirty *dirty = vbo->dirty;
311b7468b15SThomas Hellstrom pgoff_t start, cur, end;
312*668b2066SZack Rusin unsigned long res_start = res->guest_memory_offset;
313*668b2066SZack Rusin unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
314b7468b15SThomas Hellstrom
315b7468b15SThomas Hellstrom WARN_ON_ONCE(res_start & ~PAGE_MASK);
316b7468b15SThomas Hellstrom res_start >>= PAGE_SHIFT;
317b7468b15SThomas Hellstrom res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
318b7468b15SThomas Hellstrom
319b7468b15SThomas Hellstrom if (res_start >= dirty->end || res_end <= dirty->start)
320b7468b15SThomas Hellstrom return;
321b7468b15SThomas Hellstrom
322b7468b15SThomas Hellstrom cur = max(res_start, dirty->start);
323b7468b15SThomas Hellstrom res_end = max(res_end, dirty->end);
324b7468b15SThomas Hellstrom while (cur < res_end) {
325b7468b15SThomas Hellstrom unsigned long num;
326b7468b15SThomas Hellstrom
327b7468b15SThomas Hellstrom start = find_next_bit(&dirty->bitmap[0], res_end, cur);
328b7468b15SThomas Hellstrom if (start >= res_end)
329b7468b15SThomas Hellstrom break;
330b7468b15SThomas Hellstrom
331b7468b15SThomas Hellstrom end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
332b7468b15SThomas Hellstrom cur = end + 1;
333b7468b15SThomas Hellstrom num = end - start;
334b7468b15SThomas Hellstrom bitmap_clear(&dirty->bitmap[0], start, num);
335b7468b15SThomas Hellstrom vmw_resource_dirty_update(res, start, end);
336b7468b15SThomas Hellstrom }
337b7468b15SThomas Hellstrom
338b7468b15SThomas Hellstrom if (res_start <= dirty->start && res_end > dirty->start)
339b7468b15SThomas Hellstrom dirty->start = res_end;
340b7468b15SThomas Hellstrom if (res_start < dirty->end && res_end >= dirty->end)
341b7468b15SThomas Hellstrom dirty->end = res_start;
342b7468b15SThomas Hellstrom }
343b7468b15SThomas Hellstrom
344b7468b15SThomas Hellstrom /**
345b7468b15SThomas Hellstrom * vmw_bo_dirty_clear_res - Clear a resource's dirty region from
346b7468b15SThomas Hellstrom * its backing mob.
347b7468b15SThomas Hellstrom * @res: The resource
348b7468b15SThomas Hellstrom *
349b7468b15SThomas Hellstrom * This function will clear all dirty ranges affecting the resource from
350b7468b15SThomas Hellstrom * it's backup mob's dirty tracking.
351b7468b15SThomas Hellstrom */
vmw_bo_dirty_clear_res(struct vmw_resource * res)352b7468b15SThomas Hellstrom void vmw_bo_dirty_clear_res(struct vmw_resource *res)
353b7468b15SThomas Hellstrom {
354*668b2066SZack Rusin unsigned long res_start = res->guest_memory_offset;
355*668b2066SZack Rusin unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
356*668b2066SZack Rusin struct vmw_bo *vbo = res->guest_memory_bo;
357b7468b15SThomas Hellstrom struct vmw_bo_dirty *dirty = vbo->dirty;
358b7468b15SThomas Hellstrom
359b7468b15SThomas Hellstrom res_start >>= PAGE_SHIFT;
360b7468b15SThomas Hellstrom res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
361b7468b15SThomas Hellstrom
362b7468b15SThomas Hellstrom if (res_start >= dirty->end || res_end <= dirty->start)
363b7468b15SThomas Hellstrom return;
364b7468b15SThomas Hellstrom
365b7468b15SThomas Hellstrom res_start = max(res_start, dirty->start);
366b7468b15SThomas Hellstrom res_end = min(res_end, dirty->end);
367b7468b15SThomas Hellstrom bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start);
368b7468b15SThomas Hellstrom
369b7468b15SThomas Hellstrom if (res_start <= dirty->start && res_end > dirty->start)
370b7468b15SThomas Hellstrom dirty->start = res_end;
371b7468b15SThomas Hellstrom if (res_start < dirty->end && res_end >= dirty->end)
372b7468b15SThomas Hellstrom dirty->end = res_start;
373b7468b15SThomas Hellstrom }
374b7468b15SThomas Hellstrom
vmw_bo_vm_mkwrite(struct vm_fault * vmf)375b7468b15SThomas Hellstrom vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf)
376b7468b15SThomas Hellstrom {
377b7468b15SThomas Hellstrom struct vm_area_struct *vma = vmf->vma;
378b7468b15SThomas Hellstrom struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
379b7468b15SThomas Hellstrom vma->vm_private_data;
380b7468b15SThomas Hellstrom vm_fault_t ret;
381b7468b15SThomas Hellstrom unsigned long page_offset;
382b7468b15SThomas Hellstrom unsigned int save_flags;
383*668b2066SZack Rusin struct vmw_bo *vbo = to_vmw_bo(&bo->base);
384b7468b15SThomas Hellstrom
385b7468b15SThomas Hellstrom /*
386b7468b15SThomas Hellstrom * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly.
387b7468b15SThomas Hellstrom * So make sure the TTM helpers are aware.
388b7468b15SThomas Hellstrom */
389b7468b15SThomas Hellstrom save_flags = vmf->flags;
390b7468b15SThomas Hellstrom vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY;
391b7468b15SThomas Hellstrom ret = ttm_bo_vm_reserve(bo, vmf);
392b7468b15SThomas Hellstrom vmf->flags = save_flags;
393b7468b15SThomas Hellstrom if (ret)
394b7468b15SThomas Hellstrom return ret;
395b7468b15SThomas Hellstrom
396b7468b15SThomas Hellstrom page_offset = vmf->pgoff - drm_vma_node_start(&bo->base.vma_node);
397e3c92eb4SSomalapuram Amaranath if (unlikely(page_offset >= PFN_UP(bo->resource->size))) {
398b7468b15SThomas Hellstrom ret = VM_FAULT_SIGBUS;
399b7468b15SThomas Hellstrom goto out_unlock;
400b7468b15SThomas Hellstrom }
401b7468b15SThomas Hellstrom
402b7468b15SThomas Hellstrom if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE &&
403b7468b15SThomas Hellstrom !test_bit(page_offset, &vbo->dirty->bitmap[0])) {
404b7468b15SThomas Hellstrom struct vmw_bo_dirty *dirty = vbo->dirty;
405b7468b15SThomas Hellstrom
406b7468b15SThomas Hellstrom __set_bit(page_offset, &dirty->bitmap[0]);
407b7468b15SThomas Hellstrom dirty->start = min(dirty->start, page_offset);
408b7468b15SThomas Hellstrom dirty->end = max(dirty->end, page_offset + 1);
409b7468b15SThomas Hellstrom }
410b7468b15SThomas Hellstrom
411b7468b15SThomas Hellstrom out_unlock:
412b7468b15SThomas Hellstrom dma_resv_unlock(bo->base.resv);
413b7468b15SThomas Hellstrom return ret;
414b7468b15SThomas Hellstrom }
415b7468b15SThomas Hellstrom
vmw_bo_vm_fault(struct vm_fault * vmf)416b7468b15SThomas Hellstrom vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
417b7468b15SThomas Hellstrom {
418b7468b15SThomas Hellstrom struct vm_area_struct *vma = vmf->vma;
419b7468b15SThomas Hellstrom struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
420b7468b15SThomas Hellstrom vma->vm_private_data;
421*668b2066SZack Rusin struct vmw_bo *vbo = to_vmw_bo(&bo->base);
422b7468b15SThomas Hellstrom pgoff_t num_prefault;
423b7468b15SThomas Hellstrom pgprot_t prot;
424b7468b15SThomas Hellstrom vm_fault_t ret;
425b7468b15SThomas Hellstrom
426b7468b15SThomas Hellstrom ret = ttm_bo_vm_reserve(bo, vmf);
427b7468b15SThomas Hellstrom if (ret)
428b7468b15SThomas Hellstrom return ret;
429b7468b15SThomas Hellstrom
430fb80edb0SThomas Hellstrom num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 :
431fb80edb0SThomas Hellstrom TTM_BO_VM_NUM_PREFAULT;
432fb80edb0SThomas Hellstrom
433fb80edb0SThomas Hellstrom if (vbo->dirty) {
434fb80edb0SThomas Hellstrom pgoff_t allowed_prefault;
435fb80edb0SThomas Hellstrom unsigned long page_offset;
436fb80edb0SThomas Hellstrom
437fb80edb0SThomas Hellstrom page_offset = vmf->pgoff -
438fb80edb0SThomas Hellstrom drm_vma_node_start(&bo->base.vma_node);
439e3c92eb4SSomalapuram Amaranath if (page_offset >= PFN_UP(bo->resource->size) ||
440fb80edb0SThomas Hellstrom vmw_resources_clean(vbo, page_offset,
441fb80edb0SThomas Hellstrom page_offset + PAGE_SIZE,
442fb80edb0SThomas Hellstrom &allowed_prefault)) {
443fb80edb0SThomas Hellstrom ret = VM_FAULT_SIGBUS;
444fb80edb0SThomas Hellstrom goto out_unlock;
445fb80edb0SThomas Hellstrom }
446fb80edb0SThomas Hellstrom
447fb80edb0SThomas Hellstrom num_prefault = min(num_prefault, allowed_prefault);
448fb80edb0SThomas Hellstrom }
449fb80edb0SThomas Hellstrom
450b7468b15SThomas Hellstrom /*
451fb80edb0SThomas Hellstrom * If we don't track dirty using the MKWRITE method, make sure
452fb80edb0SThomas Hellstrom * sure the page protection is write-enabled so we don't get
453fb80edb0SThomas Hellstrom * a lot of unnecessary write faults.
454b7468b15SThomas Hellstrom */
455b7468b15SThomas Hellstrom if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
45675390281SThomas Hellstrom (VMware) prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
457b7468b15SThomas Hellstrom else
458b7468b15SThomas Hellstrom prot = vm_get_page_prot(vma->vm_flags);
459b7468b15SThomas Hellstrom
4600d979509SJason Gunthorpe ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
461b7468b15SThomas Hellstrom if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
462b7468b15SThomas Hellstrom return ret;
463b7468b15SThomas Hellstrom
464fb80edb0SThomas Hellstrom out_unlock:
465b7468b15SThomas Hellstrom dma_resv_unlock(bo->base.resv);
466fb80edb0SThomas Hellstrom
467b7468b15SThomas Hellstrom return ret;
468b7468b15SThomas Hellstrom }
469