1/*
  2 * Copyright © 2016 Intel Corporation
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice (including the next
 12 * paragraph) shall be included in all copies or substantial portions of the
 13 * Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 21 * IN THE SOFTWARE.
 22 *
 23 */
 24
 25#include <linux/kernel.h>
 26#include <asm/fpu/api.h>
 27
 28#include "i915_memcpy.h"
 29
 30static DEFINE_STATIC_KEY_FALSE(has_movntdqa);
 31
 32#ifdef CONFIG_AS_MOVNTDQA
 33static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
 34{
 35	kernel_fpu_begin();
 36
 37	len >>= 4;
 38	while (len >= 4) {
 39		asm("movntdqa   (%0), %%xmm0\n"
 40		    "movntdqa 16(%0), %%xmm1\n"
 41		    "movntdqa 32(%0), %%xmm2\n"
 42		    "movntdqa 48(%0), %%xmm3\n"
 43		    "movaps %%xmm0,   (%1)\n"
 44		    "movaps %%xmm1, 16(%1)\n"
 45		    "movaps %%xmm2, 32(%1)\n"
 46		    "movaps %%xmm3, 48(%1)\n"
 47		    :: "r" (src), "r" (dst) : "memory");
 48		src += 64;
 49		dst += 64;
 50		len -= 4;
 51	}
 52	while (len--) {
 53		asm("movntdqa (%0), %%xmm0\n"
 54		    "movaps %%xmm0, (%1)\n"
 55		    :: "r" (src), "r" (dst) : "memory");
 56		src += 16;
 57		dst += 16;
 58	}
 59
 60	kernel_fpu_end();
 61}
 62#endif
 63
 64/**
 65 * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC
 66 * @dst: destination pointer
 67 * @src: source pointer
 68 * @len: how many bytes to copy
 69 *
 70 * i915_memcpy_from_wc copies @len bytes from @src to @dst using
 71 * non-temporal instructions where available. Note that all arguments
 72 * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
 73 * of 16.
 74 *
 75 * To test whether accelerated reads from WC are supported, use
 76 * i915_memcpy_from_wc(NULL, NULL, 0);
 77 *
 78 * Returns true if the copy was successful, false if the preconditions
 79 * are not met.
 80 */
 81bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
 82{
 83	if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))
 84		return false;
 85
 86#ifdef CONFIG_AS_MOVNTDQA
 87	if (static_branch_likely(&has_movntdqa)) {
 88		if (likely(len))
 89			__memcpy_ntdqa(dst, src, len);
 90		return true;
 91	}
 92#endif
 93
 94	return false;
 95}
 96
 97void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
 98{
 99	/*
100	 * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions
101	 * emulation. So don't enable movntdqa in hypervisor guest.
102	 */
103	if (static_cpu_has(X86_FEATURE_XMM4_1) &&
104	    !boot_cpu_has(X86_FEATURE_HYPERVISOR))
105		static_branch_enable(&has_movntdqa);
106}