1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
3  *
4  * Copyright 2019 VMware, Inc., Palo Alto, CA., USA
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 #include "vmwgfx_drv.h"
28 
29 /*
30  * Different methods for tracking dirty:
31  * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits
32  * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write-
33  * accesses in the VM mkwrite() callback
34  */
35 enum vmw_bo_dirty_method {
36 	VMW_BO_DIRTY_PAGETABLE,
37 	VMW_BO_DIRTY_MKWRITE,
38 };
39 
40 /*
41  * No dirtied pages at scan trigger a transition to the _MKWRITE method,
42  * similarly a certain percentage of dirty pages trigger a transition to
43  * the _PAGETABLE method. How many triggers should we wait for before
44  * changing method?
45  */
46 #define VMW_DIRTY_NUM_CHANGE_TRIGGERS 2
47 
48 /* Percentage to trigger a transition to the _PAGETABLE method */
49 #define VMW_DIRTY_PERCENTAGE 10
50 
51 /**
52  * struct vmw_bo_dirty - Dirty information for buffer objects
53  * @start: First currently dirty bit
54  * @end: Last currently dirty bit + 1
55  * @method: The currently used dirty method
56  * @change_count: Number of consecutive method change triggers
57  * @ref_count: Reference count for this structure
58  * @bitmap_size: The size of the bitmap in bits. Typically equal to the
59  * nuber of pages in the bo.
60  * @bitmap: A bitmap where each bit represents a page. A set bit means a
61  * dirty page.
62  */
63 struct vmw_bo_dirty {
64 	unsigned long start;
65 	unsigned long end;
66 	enum vmw_bo_dirty_method method;
67 	unsigned int change_count;
68 	unsigned int ref_count;
69 	unsigned long bitmap_size;
70 	unsigned long bitmap[];
71 };
72 
73 /**
74  * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits
75  * @vbo: The buffer object to scan
76  *
77  * Scans the pagetable for dirty bits. Clear those bits and modify the
78  * dirty structure with the results. This function may change the
79  * dirty-tracking method.
80  */
81 static void vmw_bo_dirty_scan_pagetable(struct vmw_buffer_object *vbo)
82 {
83 	struct vmw_bo_dirty *dirty = vbo->dirty;
84 	pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node);
85 	struct address_space *mapping = vbo->base.bdev->dev_mapping;
86 	pgoff_t num_marked;
87 
88 	num_marked = clean_record_shared_mapping_range
89 		(mapping,
90 		 offset, dirty->bitmap_size,
91 		 offset, &dirty->bitmap[0],
92 		 &dirty->start, &dirty->end);
93 	if (num_marked == 0)
94 		dirty->change_count++;
95 	else
96 		dirty->change_count = 0;
97 
98 	if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
99 		dirty->change_count = 0;
100 		dirty->method = VMW_BO_DIRTY_MKWRITE;
101 		wp_shared_mapping_range(mapping,
102 					offset, dirty->bitmap_size);
103 		clean_record_shared_mapping_range(mapping,
104 						  offset, dirty->bitmap_size,
105 						  offset, &dirty->bitmap[0],
106 						  &dirty->start, &dirty->end);
107 	}
108 }
109 
110 /**
111  * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method
112  * @vbo: The buffer object to scan
113  *
114  * Write-protect pages written to so that consecutive write accesses will
115  * trigger a call to mkwrite.
116  *
117  * This function may change the dirty-tracking method.
118  */
119 static void vmw_bo_dirty_scan_mkwrite(struct vmw_buffer_object *vbo)
120 {
121 	struct vmw_bo_dirty *dirty = vbo->dirty;
122 	unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
123 	struct address_space *mapping = vbo->base.bdev->dev_mapping;
124 	pgoff_t num_marked;
125 
126 	if (dirty->end <= dirty->start)
127 		return;
128 
129 	num_marked = wp_shared_mapping_range(vbo->base.bdev->dev_mapping,
130 					dirty->start + offset,
131 					dirty->end - dirty->start);
132 
133 	if (100UL * num_marked / dirty->bitmap_size >
134 	    VMW_DIRTY_PERCENTAGE) {
135 		dirty->change_count++;
136 	} else {
137 		dirty->change_count = 0;
138 	}
139 
140 	if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
141 		pgoff_t start = 0;
142 		pgoff_t end = dirty->bitmap_size;
143 
144 		dirty->method = VMW_BO_DIRTY_PAGETABLE;
145 		clean_record_shared_mapping_range(mapping, offset, end, offset,
146 						  &dirty->bitmap[0],
147 						  &start, &end);
148 		bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size);
149 		if (dirty->start < dirty->end)
150 			bitmap_set(&dirty->bitmap[0], dirty->start,
151 				   dirty->end - dirty->start);
152 		dirty->change_count = 0;
153 	}
154 }
155 
156 /**
157  * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty
158  * tracking structure
159  * @vbo: The buffer object to scan
160  *
161  * This function may change the dirty tracking method.
162  */
163 void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo)
164 {
165 	struct vmw_bo_dirty *dirty = vbo->dirty;
166 
167 	if (dirty->method == VMW_BO_DIRTY_PAGETABLE)
168 		vmw_bo_dirty_scan_pagetable(vbo);
169 	else
170 		vmw_bo_dirty_scan_mkwrite(vbo);
171 }
172 
173 /**
174  * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before
175  * an unmap_mapping_range operation.
176  * @vbo: The buffer object,
177  * @start: First page of the range within the buffer object.
178  * @end: Last page of the range within the buffer object + 1.
179  *
180  * If we're using the _PAGETABLE scan method, we may leak dirty pages
181  * when calling unmap_mapping_range(). This function makes sure we pick
182  * up all dirty pages.
183  */
184 static void vmw_bo_dirty_pre_unmap(struct vmw_buffer_object *vbo,
185 				   pgoff_t start, pgoff_t end)
186 {
187 	struct vmw_bo_dirty *dirty = vbo->dirty;
188 	unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
189 	struct address_space *mapping = vbo->base.bdev->dev_mapping;
190 
191 	if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end)
192 		return;
193 
194 	wp_shared_mapping_range(mapping, start + offset, end - start);
195 	clean_record_shared_mapping_range(mapping, start + offset,
196 					  end - start, offset,
197 					  &dirty->bitmap[0], &dirty->start,
198 					  &dirty->end);
199 }
200 
201 /**
202  * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo
203  * @vbo: The buffer object,
204  * @start: First page of the range within the buffer object.
205  * @end: Last page of the range within the buffer object + 1.
206  *
207  * This is similar to ttm_bo_unmap_virtual() except it takes a subrange.
208  */
209 void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo,
210 			pgoff_t start, pgoff_t end)
211 {
212 	unsigned long offset = drm_vma_node_start(&vbo->base.base.vma_node);
213 	struct address_space *mapping = vbo->base.bdev->dev_mapping;
214 
215 	vmw_bo_dirty_pre_unmap(vbo, start, end);
216 	unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT,
217 				   (loff_t) (end - start) << PAGE_SHIFT);
218 }
219 
220 /**
221  * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object
222  * @vbo: The buffer object
223  *
224  * This function registers a dirty-tracking user to a buffer object.
225  * A user can be for example a resource or a vma in a special user-space
226  * mapping.
227  *
228  * Return: Zero on success, -ENOMEM on memory allocation failure.
229  */
230 int vmw_bo_dirty_add(struct vmw_buffer_object *vbo)
231 {
232 	struct vmw_bo_dirty *dirty = vbo->dirty;
233 	pgoff_t num_pages = vbo->base.resource->num_pages;
234 	size_t size;
235 	int ret;
236 
237 	if (dirty) {
238 		dirty->ref_count++;
239 		return 0;
240 	}
241 
242 	size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long);
243 	dirty = kvzalloc(size, GFP_KERNEL);
244 	if (!dirty) {
245 		ret = -ENOMEM;
246 		goto out_no_dirty;
247 	}
248 
249 	dirty->bitmap_size = num_pages;
250 	dirty->start = dirty->bitmap_size;
251 	dirty->end = 0;
252 	dirty->ref_count = 1;
253 	if (num_pages < PAGE_SIZE / sizeof(pte_t)) {
254 		dirty->method = VMW_BO_DIRTY_PAGETABLE;
255 	} else {
256 		struct address_space *mapping = vbo->base.bdev->dev_mapping;
257 		pgoff_t offset = drm_vma_node_start(&vbo->base.base.vma_node);
258 
259 		dirty->method = VMW_BO_DIRTY_MKWRITE;
260 
261 		/* Write-protect and then pick up already dirty bits */
262 		wp_shared_mapping_range(mapping, offset, num_pages);
263 		clean_record_shared_mapping_range(mapping, offset, num_pages,
264 						  offset,
265 						  &dirty->bitmap[0],
266 						  &dirty->start, &dirty->end);
267 	}
268 
269 	vbo->dirty = dirty;
270 
271 	return 0;
272 
273 out_no_dirty:
274 	return ret;
275 }
276 
277 /**
278  * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object
279  * @vbo: The buffer object
280  *
281  * This function releases a dirty-tracking user from a buffer object.
282  * If the reference count reaches zero, then the dirty-tracking object is
283  * freed and the pointer to it cleared.
284  *
285  * Return: Zero on success, -ENOMEM on memory allocation failure.
286  */
287 void vmw_bo_dirty_release(struct vmw_buffer_object *vbo)
288 {
289 	struct vmw_bo_dirty *dirty = vbo->dirty;
290 
291 	if (dirty && --dirty->ref_count == 0) {
292 		kvfree(dirty);
293 		vbo->dirty = NULL;
294 	}
295 }
296 
297 /**
298  * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from
299  * its backing mob.
300  * @res: The resource
301  *
302  * This function will pick up all dirty ranges affecting the resource from
303  * it's backup mob, and call vmw_resource_dirty_update() once for each
304  * range. The transferred ranges will be cleared from the backing mob's
305  * dirty tracking.
306  */
307 void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)
308 {
309 	struct vmw_buffer_object *vbo = res->backup;
310 	struct vmw_bo_dirty *dirty = vbo->dirty;
311 	pgoff_t start, cur, end;
312 	unsigned long res_start = res->backup_offset;
313 	unsigned long res_end = res->backup_offset + res->backup_size;
314 
315 	WARN_ON_ONCE(res_start & ~PAGE_MASK);
316 	res_start >>= PAGE_SHIFT;
317 	res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
318 
319 	if (res_start >= dirty->end || res_end <= dirty->start)
320 		return;
321 
322 	cur = max(res_start, dirty->start);
323 	res_end = max(res_end, dirty->end);
324 	while (cur < res_end) {
325 		unsigned long num;
326 
327 		start = find_next_bit(&dirty->bitmap[0], res_end, cur);
328 		if (start >= res_end)
329 			break;
330 
331 		end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
332 		cur = end + 1;
333 		num = end - start;
334 		bitmap_clear(&dirty->bitmap[0], start, num);
335 		vmw_resource_dirty_update(res, start, end);
336 	}
337 
338 	if (res_start <= dirty->start && res_end > dirty->start)
339 		dirty->start = res_end;
340 	if (res_start < dirty->end && res_end >= dirty->end)
341 		dirty->end = res_start;
342 }
343 
344 /**
345  * vmw_bo_dirty_clear_res - Clear a resource's dirty region from
346  * its backing mob.
347  * @res: The resource
348  *
349  * This function will clear all dirty ranges affecting the resource from
350  * it's backup mob's dirty tracking.
351  */
352 void vmw_bo_dirty_clear_res(struct vmw_resource *res)
353 {
354 	unsigned long res_start = res->backup_offset;
355 	unsigned long res_end = res->backup_offset + res->backup_size;
356 	struct vmw_buffer_object *vbo = res->backup;
357 	struct vmw_bo_dirty *dirty = vbo->dirty;
358 
359 	res_start >>= PAGE_SHIFT;
360 	res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
361 
362 	if (res_start >= dirty->end || res_end <= dirty->start)
363 		return;
364 
365 	res_start = max(res_start, dirty->start);
366 	res_end = min(res_end, dirty->end);
367 	bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start);
368 
369 	if (res_start <= dirty->start && res_end > dirty->start)
370 		dirty->start = res_end;
371 	if (res_start < dirty->end && res_end >= dirty->end)
372 		dirty->end = res_start;
373 }
374 
375 vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf)
376 {
377 	struct vm_area_struct *vma = vmf->vma;
378 	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
379 	    vma->vm_private_data;
380 	vm_fault_t ret;
381 	unsigned long page_offset;
382 	unsigned int save_flags;
383 	struct vmw_buffer_object *vbo =
384 		container_of(bo, typeof(*vbo), base);
385 
386 	/*
387 	 * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly.
388 	 * So make sure the TTM helpers are aware.
389 	 */
390 	save_flags = vmf->flags;
391 	vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY;
392 	ret = ttm_bo_vm_reserve(bo, vmf);
393 	vmf->flags = save_flags;
394 	if (ret)
395 		return ret;
396 
397 	page_offset = vmf->pgoff - drm_vma_node_start(&bo->base.vma_node);
398 	if (unlikely(page_offset >= bo->resource->num_pages)) {
399 		ret = VM_FAULT_SIGBUS;
400 		goto out_unlock;
401 	}
402 
403 	if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE &&
404 	    !test_bit(page_offset, &vbo->dirty->bitmap[0])) {
405 		struct vmw_bo_dirty *dirty = vbo->dirty;
406 
407 		__set_bit(page_offset, &dirty->bitmap[0]);
408 		dirty->start = min(dirty->start, page_offset);
409 		dirty->end = max(dirty->end, page_offset + 1);
410 	}
411 
412 out_unlock:
413 	dma_resv_unlock(bo->base.resv);
414 	return ret;
415 }
416 
417 vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
418 {
419 	struct vm_area_struct *vma = vmf->vma;
420 	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
421 	    vma->vm_private_data;
422 	struct vmw_buffer_object *vbo =
423 		container_of(bo, struct vmw_buffer_object, base);
424 	pgoff_t num_prefault;
425 	pgprot_t prot;
426 	vm_fault_t ret;
427 
428 	ret = ttm_bo_vm_reserve(bo, vmf);
429 	if (ret)
430 		return ret;
431 
432 	num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 :
433 		TTM_BO_VM_NUM_PREFAULT;
434 
435 	if (vbo->dirty) {
436 		pgoff_t allowed_prefault;
437 		unsigned long page_offset;
438 
439 		page_offset = vmf->pgoff -
440 			drm_vma_node_start(&bo->base.vma_node);
441 		if (page_offset >= bo->resource->num_pages ||
442 		    vmw_resources_clean(vbo, page_offset,
443 					page_offset + PAGE_SIZE,
444 					&allowed_prefault)) {
445 			ret = VM_FAULT_SIGBUS;
446 			goto out_unlock;
447 		}
448 
449 		num_prefault = min(num_prefault, allowed_prefault);
450 	}
451 
452 	/*
453 	 * If we don't track dirty using the MKWRITE method, make sure
454 	 * sure the page protection is write-enabled so we don't get
455 	 * a lot of unnecessary write faults.
456 	 */
457 	if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
458 		prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
459 	else
460 		prot = vm_get_page_prot(vma->vm_flags);
461 
462 	ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
463 	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
464 		return ret;
465 
466 out_unlock:
467 	dma_resv_unlock(bo->base.resv);
468 
469 	return ret;
470 }
471