1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /************************************************************************** 3 * 4 * Copyright 2017 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 29 #include "vmwgfx_drv.h" 30 31 /* 32 * Template that implements find_first_diff() for a generic 33 * unsigned integer type. @size and return value are in bytes. 34 */ 35 #define VMW_FIND_FIRST_DIFF(_type) \ 36 static size_t vmw_find_first_diff_ ## _type \ 37 (const _type * dst, const _type * src, size_t size)\ 38 { \ 39 size_t i; \ 40 \ 41 for (i = 0; i < size; i += sizeof(_type)) { \ 42 if (*dst++ != *src++) \ 43 break; \ 44 } \ 45 \ 46 return i; \ 47 } 48 49 50 /* 51 * Template that implements find_last_diff() for a generic 52 * unsigned integer type. Pointers point to the item following the 53 * *end* of the area to be examined. @size and return value are in 54 * bytes. 55 */ 56 #define VMW_FIND_LAST_DIFF(_type) \ 57 static ssize_t vmw_find_last_diff_ ## _type( \ 58 const _type * dst, const _type * src, size_t size) \ 59 { \ 60 while (size) { \ 61 if (*--dst != *--src) \ 62 break; \ 63 \ 64 size -= sizeof(_type); \ 65 } \ 66 return size; \ 67 } 68 69 70 /* 71 * Instantiate find diff functions for relevant unsigned integer sizes, 72 * assuming that wider integers are faster (including aligning) up to the 73 * architecture native width, which is assumed to be 32 bit unless 74 * CONFIG_64BIT is defined. 75 */ 76 VMW_FIND_FIRST_DIFF(u8); 77 VMW_FIND_LAST_DIFF(u8); 78 79 VMW_FIND_FIRST_DIFF(u16); 80 VMW_FIND_LAST_DIFF(u16); 81 82 VMW_FIND_FIRST_DIFF(u32); 83 VMW_FIND_LAST_DIFF(u32); 84 85 #ifdef CONFIG_64BIT 86 VMW_FIND_FIRST_DIFF(u64); 87 VMW_FIND_LAST_DIFF(u64); 88 #endif 89 90 91 /* We use size aligned copies. This computes (addr - align(addr)) */ 92 #define SPILL(_var, _type) ((unsigned long) _var & (sizeof(_type) - 1)) 93 94 95 /* 96 * Template to compute find_first_diff() for a certain integer type 97 * including a head copy for alignment, and adjustment of parameters 98 * for tail find or increased resolution find using an unsigned integer find 99 * of smaller width. If finding is complete, and resolution is sufficient, 100 * the macro executes a return statement. Otherwise it falls through. 101 */ 102 #define VMW_TRY_FIND_FIRST_DIFF(_type) \ 103 do { \ 104 unsigned int spill = SPILL(dst, _type); \ 105 size_t diff_offs; \ 106 \ 107 if (spill && spill == SPILL(src, _type) && \ 108 sizeof(_type) - spill <= size) { \ 109 spill = sizeof(_type) - spill; \ 110 diff_offs = vmw_find_first_diff_u8(dst, src, spill); \ 111 if (diff_offs < spill) \ 112 return round_down(offset + diff_offs, granularity); \ 113 \ 114 dst += spill; \ 115 src += spill; \ 116 size -= spill; \ 117 offset += spill; \ 118 spill = 0; \ 119 } \ 120 if (!spill && !SPILL(src, _type)) { \ 121 size_t to_copy = size & ~(sizeof(_type) - 1); \ 122 \ 123 diff_offs = vmw_find_first_diff_ ## _type \ 124 ((_type *) dst, (_type *) src, to_copy); \ 125 if (diff_offs >= size || granularity == sizeof(_type)) \ 126 return (offset + diff_offs); \ 127 \ 128 dst += diff_offs; \ 129 src += diff_offs; \ 130 size -= diff_offs; \ 131 offset += diff_offs; \ 132 } \ 133 } while (0) \ 134 135 136 /** 137 * vmw_find_first_diff - find the first difference between dst and src 138 * 139 * @dst: The destination address 140 * @src: The source address 141 * @size: Number of bytes to compare 142 * @granularity: The granularity needed for the return value in bytes. 143 * return: The offset from find start where the first difference was 144 * encountered in bytes. If no difference was found, the function returns 145 * a value >= @size. 146 */ 147 static size_t vmw_find_first_diff(const u8 *dst, const u8 *src, size_t size, 148 size_t granularity) 149 { 150 size_t offset = 0; 151 152 /* 153 * Try finding with large integers if alignment allows, or we can 154 * fix it. Fall through if we need better resolution or alignment 155 * was bad. 156 */ 157 #ifdef CONFIG_64BIT 158 VMW_TRY_FIND_FIRST_DIFF(u64); 159 #endif 160 VMW_TRY_FIND_FIRST_DIFF(u32); 161 VMW_TRY_FIND_FIRST_DIFF(u16); 162 163 return round_down(offset + vmw_find_first_diff_u8(dst, src, size), 164 granularity); 165 } 166 167 168 /* 169 * Template to compute find_last_diff() for a certain integer type 170 * including a tail copy for alignment, and adjustment of parameters 171 * for head find or increased resolution find using an unsigned integer find 172 * of smaller width. If finding is complete, and resolution is sufficient, 173 * the macro executes a return statement. Otherwise it falls through. 174 */ 175 #define VMW_TRY_FIND_LAST_DIFF(_type) \ 176 do { \ 177 unsigned int spill = SPILL(dst, _type); \ 178 ssize_t location; \ 179 ssize_t diff_offs; \ 180 \ 181 if (spill && spill <= size && spill == SPILL(src, _type)) { \ 182 diff_offs = vmw_find_last_diff_u8(dst, src, spill); \ 183 if (diff_offs) { \ 184 location = size - spill + diff_offs - 1; \ 185 return round_down(location, granularity); \ 186 } \ 187 \ 188 dst -= spill; \ 189 src -= spill; \ 190 size -= spill; \ 191 spill = 0; \ 192 } \ 193 if (!spill && !SPILL(src, _type)) { \ 194 size_t to_copy = round_down(size, sizeof(_type)); \ 195 \ 196 diff_offs = vmw_find_last_diff_ ## _type \ 197 ((_type *) dst, (_type *) src, to_copy); \ 198 location = size - to_copy + diff_offs - sizeof(_type); \ 199 if (location < 0 || granularity == sizeof(_type)) \ 200 return location; \ 201 \ 202 dst -= to_copy - diff_offs; \ 203 src -= to_copy - diff_offs; \ 204 size -= to_copy - diff_offs; \ 205 } \ 206 } while (0) 207 208 209 /** 210 * vmw_find_last_diff - find the last difference between dst and src 211 * 212 * @dst: The destination address 213 * @src: The source address 214 * @size: Number of bytes to compare 215 * @granularity: The granularity needed for the return value in bytes. 216 * return: The offset from find start where the last difference was 217 * encountered in bytes, or a negative value if no difference was found. 218 */ 219 static ssize_t vmw_find_last_diff(const u8 *dst, const u8 *src, size_t size, 220 size_t granularity) 221 { 222 dst += size; 223 src += size; 224 225 #ifdef CONFIG_64BIT 226 VMW_TRY_FIND_LAST_DIFF(u64); 227 #endif 228 VMW_TRY_FIND_LAST_DIFF(u32); 229 VMW_TRY_FIND_LAST_DIFF(u16); 230 231 return round_down(vmw_find_last_diff_u8(dst, src, size) - 1, 232 granularity); 233 } 234 235 236 /** 237 * vmw_memcpy - A wrapper around kernel memcpy with allowing to plug it into a 238 * struct vmw_diff_cpy. 239 * 240 * @diff: The struct vmw_diff_cpy closure argument (unused). 241 * @dest: The copy destination. 242 * @src: The copy source. 243 * @n: Number of bytes to copy. 244 */ 245 void vmw_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, size_t n) 246 { 247 memcpy(dest, src, n); 248 } 249 250 251 /** 252 * vmw_adjust_rect - Adjust rectangle coordinates for newly found difference 253 * 254 * @diff: The struct vmw_diff_cpy used to track the modified bounding box. 255 * @diff_offs: The offset from @diff->line_offset where the difference was 256 * found. 257 */ 258 static void vmw_adjust_rect(struct vmw_diff_cpy *diff, size_t diff_offs) 259 { 260 size_t offs = (diff_offs + diff->line_offset) / diff->cpp; 261 struct drm_rect *rect = &diff->rect; 262 263 rect->x1 = min_t(int, rect->x1, offs); 264 rect->x2 = max_t(int, rect->x2, offs + 1); 265 rect->y1 = min_t(int, rect->y1, diff->line); 266 rect->y2 = max_t(int, rect->y2, diff->line + 1); 267 } 268 269 /** 270 * vmw_diff_memcpy - memcpy that creates a bounding box of modified content. 271 * 272 * @diff: The struct vmw_diff_cpy used to track the modified bounding box. 273 * @dest: The copy destination. 274 * @src: The copy source. 275 * @n: Number of bytes to copy. 276 * 277 * In order to correctly track the modified content, the field @diff->line must 278 * be pre-loaded with the current line number, the field @diff->line_offset must 279 * be pre-loaded with the line offset in bytes where the copy starts, and 280 * finally the field @diff->cpp need to be preloaded with the number of bytes 281 * per unit in the horizontal direction of the area we're examining. 282 * Typically bytes per pixel. 283 * This is needed to know the needed granularity of the difference computing 284 * operations. A higher cpp generally leads to faster execution at the cost of 285 * bounding box width precision. 286 */ 287 void vmw_diff_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, 288 size_t n) 289 { 290 ssize_t csize, byte_len; 291 292 if (WARN_ON_ONCE(round_down(n, diff->cpp) != n)) 293 return; 294 295 /* TODO: Possibly use a single vmw_find_first_diff per line? */ 296 csize = vmw_find_first_diff(dest, src, n, diff->cpp); 297 if (csize < n) { 298 vmw_adjust_rect(diff, csize); 299 byte_len = diff->cpp; 300 301 /* 302 * Starting from where first difference was found, find 303 * location of last difference, and then copy. 304 */ 305 diff->line_offset += csize; 306 dest += csize; 307 src += csize; 308 n -= csize; 309 csize = vmw_find_last_diff(dest, src, n, diff->cpp); 310 if (csize >= 0) { 311 byte_len += csize; 312 vmw_adjust_rect(diff, csize); 313 } 314 memcpy(dest, src, byte_len); 315 } 316 diff->line_offset += n; 317 } 318 319 /** 320 * struct vmw_bo_blit_line_data - Convenience argument to vmw_bo_cpu_blit_line 321 * 322 * @mapped_dst: Already mapped destination page index in @dst_pages. 323 * @dst_addr: Kernel virtual address of mapped destination page. 324 * @dst_pages: Array of destination bo pages. 325 * @dst_num_pages: Number of destination bo pages. 326 * @dst_prot: Destination bo page protection. 327 * @mapped_src: Already mapped source page index in @dst_pages. 328 * @src_addr: Kernel virtual address of mapped source page. 329 * @src_pages: Array of source bo pages. 330 * @src_num_pages: Number of source bo pages. 331 * @src_prot: Source bo page protection. 332 * @diff: Struct vmw_diff_cpy, in the end forwarded to the memcpy routine. 333 */ 334 struct vmw_bo_blit_line_data { 335 u32 mapped_dst; 336 u8 *dst_addr; 337 struct page **dst_pages; 338 u32 dst_num_pages; 339 pgprot_t dst_prot; 340 u32 mapped_src; 341 u8 *src_addr; 342 struct page **src_pages; 343 u32 src_num_pages; 344 pgprot_t src_prot; 345 struct vmw_diff_cpy *diff; 346 }; 347 348 /** 349 * vmw_bo_cpu_blit_line - Blit part of a line from one bo to another. 350 * 351 * @d: Blit data as described above. 352 * @dst_offset: Destination copy start offset from start of bo. 353 * @src_offset: Source copy start offset from start of bo. 354 * @bytes_to_copy: Number of bytes to copy in this line. 355 */ 356 static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d, 357 u32 dst_offset, 358 u32 src_offset, 359 u32 bytes_to_copy) 360 { 361 struct vmw_diff_cpy *diff = d->diff; 362 363 while (bytes_to_copy) { 364 u32 copy_size = bytes_to_copy; 365 u32 dst_page = dst_offset >> PAGE_SHIFT; 366 u32 src_page = src_offset >> PAGE_SHIFT; 367 u32 dst_page_offset = dst_offset & ~PAGE_MASK; 368 u32 src_page_offset = src_offset & ~PAGE_MASK; 369 bool unmap_dst = d->dst_addr && dst_page != d->mapped_dst; 370 bool unmap_src = d->src_addr && (src_page != d->mapped_src || 371 unmap_dst); 372 373 copy_size = min_t(u32, copy_size, PAGE_SIZE - dst_page_offset); 374 copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset); 375 376 if (unmap_src) { 377 ttm_kunmap_atomic_prot(d->src_addr, d->src_prot); 378 d->src_addr = NULL; 379 } 380 381 if (unmap_dst) { 382 ttm_kunmap_atomic_prot(d->dst_addr, d->dst_prot); 383 d->dst_addr = NULL; 384 } 385 386 if (!d->dst_addr) { 387 if (WARN_ON_ONCE(dst_page >= d->dst_num_pages)) 388 return -EINVAL; 389 390 d->dst_addr = 391 ttm_kmap_atomic_prot(d->dst_pages[dst_page], 392 d->dst_prot); 393 if (!d->dst_addr) 394 return -ENOMEM; 395 396 d->mapped_dst = dst_page; 397 } 398 399 if (!d->src_addr) { 400 if (WARN_ON_ONCE(src_page >= d->src_num_pages)) 401 return -EINVAL; 402 403 d->src_addr = 404 ttm_kmap_atomic_prot(d->src_pages[src_page], 405 d->src_prot); 406 if (!d->src_addr) 407 return -ENOMEM; 408 409 d->mapped_src = src_page; 410 } 411 diff->do_cpy(diff, d->dst_addr + dst_page_offset, 412 d->src_addr + src_page_offset, copy_size); 413 414 bytes_to_copy -= copy_size; 415 dst_offset += copy_size; 416 src_offset += copy_size; 417 } 418 419 return 0; 420 } 421 422 /** 423 * ttm_bo_cpu_blit - in-kernel cpu blit. 424 * 425 * @dst: Destination buffer object. 426 * @dst_offset: Destination offset of blit start in bytes. 427 * @dst_stride: Destination stride in bytes. 428 * @src: Source buffer object. 429 * @src_offset: Source offset of blit start in bytes. 430 * @src_stride: Source stride in bytes. 431 * @w: Width of blit. 432 * @h: Height of blit. 433 * return: Zero on success. Negative error value on failure. Will print out 434 * kernel warnings on caller bugs. 435 * 436 * Performs a CPU blit from one buffer object to another avoiding a full 437 * bo vmap which may exhaust- or fragment vmalloc space. 438 * On supported architectures (x86), we're using kmap_atomic which avoids 439 * cross-processor TLB- and cache flushes and may, on non-HIGHMEM systems 440 * reference already set-up mappings. 441 * 442 * Neither of the buffer objects may be placed in PCI memory 443 * (Fixed memory in TTM terminology) when using this function. 444 */ 445 int vmw_bo_cpu_blit(struct ttm_buffer_object *dst, 446 u32 dst_offset, u32 dst_stride, 447 struct ttm_buffer_object *src, 448 u32 src_offset, u32 src_stride, 449 u32 w, u32 h, 450 struct vmw_diff_cpy *diff) 451 { 452 struct ttm_operation_ctx ctx = { 453 .interruptible = false, 454 .no_wait_gpu = false 455 }; 456 u32 j, initial_line = dst_offset / dst_stride; 457 struct vmw_bo_blit_line_data d; 458 int ret = 0; 459 460 /* Buffer objects need to be either pinned or reserved: */ 461 if (!(dst->mem.placement & TTM_PL_FLAG_NO_EVICT)) 462 reservation_object_assert_held(dst->base.resv); 463 if (!(src->mem.placement & TTM_PL_FLAG_NO_EVICT)) 464 reservation_object_assert_held(src->base.resv); 465 466 if (dst->ttm->state == tt_unpopulated) { 467 ret = dst->ttm->bdev->driver->ttm_tt_populate(dst->ttm, &ctx); 468 if (ret) 469 return ret; 470 } 471 472 if (src->ttm->state == tt_unpopulated) { 473 ret = src->ttm->bdev->driver->ttm_tt_populate(src->ttm, &ctx); 474 if (ret) 475 return ret; 476 } 477 478 d.mapped_dst = 0; 479 d.mapped_src = 0; 480 d.dst_addr = NULL; 481 d.src_addr = NULL; 482 d.dst_pages = dst->ttm->pages; 483 d.src_pages = src->ttm->pages; 484 d.dst_num_pages = dst->num_pages; 485 d.src_num_pages = src->num_pages; 486 d.dst_prot = ttm_io_prot(dst->mem.placement, PAGE_KERNEL); 487 d.src_prot = ttm_io_prot(src->mem.placement, PAGE_KERNEL); 488 d.diff = diff; 489 490 for (j = 0; j < h; ++j) { 491 diff->line = j + initial_line; 492 diff->line_offset = dst_offset % dst_stride; 493 ret = vmw_bo_cpu_blit_line(&d, dst_offset, src_offset, w); 494 if (ret) 495 goto out; 496 497 dst_offset += dst_stride; 498 src_offset += src_stride; 499 } 500 out: 501 if (d.src_addr) 502 ttm_kunmap_atomic_prot(d.src_addr, d.src_prot); 503 if (d.dst_addr) 504 ttm_kunmap_atomic_prot(d.dst_addr, d.dst_prot); 505 506 return ret; 507 } 508