1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/kernel.h> 26 #include <asm/fpu/api.h> 27 28 #include "i915_drv.h" 29 30 static DEFINE_STATIC_KEY_FALSE(has_movntdqa); 31 32 #ifdef CONFIG_AS_MOVNTDQA 33 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) 34 { 35 kernel_fpu_begin(); 36 37 len >>= 4; 38 while (len >= 4) { 39 asm("movntdqa (%0), %%xmm0\n" 40 "movntdqa 16(%0), %%xmm1\n" 41 "movntdqa 32(%0), %%xmm2\n" 42 "movntdqa 48(%0), %%xmm3\n" 43 "movaps %%xmm0, (%1)\n" 44 "movaps %%xmm1, 16(%1)\n" 45 "movaps %%xmm2, 32(%1)\n" 46 "movaps %%xmm3, 48(%1)\n" 47 :: "r" (src), "r" (dst) : "memory"); 48 src += 64; 49 dst += 64; 50 len -= 4; 51 } 52 while (len--) { 53 asm("movntdqa (%0), %%xmm0\n" 54 "movaps %%xmm0, (%1)\n" 55 :: "r" (src), "r" (dst) : "memory"); 56 src += 16; 57 dst += 16; 58 } 59 60 kernel_fpu_end(); 61 } 62 #endif 63 64 /** 65 * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC 66 * @dst: destination pointer 67 * @src: source pointer 68 * @len: how many bytes to copy 69 * 70 * i915_memcpy_from_wc copies @len bytes from @src to @dst using 71 * non-temporal instructions where available. Note that all arguments 72 * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple 73 * of 16. 74 * 75 * To test whether accelerated reads from WC are supported, use 76 * i915_memcpy_from_wc(NULL, NULL, 0); 77 * 78 * Returns true if the copy was successful, false if the preconditions 79 * are not met. 80 */ 81 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len) 82 { 83 if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15)) 84 return false; 85 86 #ifdef CONFIG_AS_MOVNTDQA 87 if (static_branch_likely(&has_movntdqa)) { 88 if (likely(len)) 89 __memcpy_ntdqa(dst, src, len); 90 return true; 91 } 92 #endif 93 94 return false; 95 } 96 97 void i915_memcpy_init_early(struct drm_i915_private *dev_priv) 98 { 99 if (static_cpu_has(X86_FEATURE_XMM4_1)) 100 static_branch_enable(&has_movntdqa); 101 } 102