1// SPDX-License-Identifier: GPL-2.0
 2/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
 3
 4#include <linux/jump_label.h>
 5#include <linux/uaccess.h>
 6#include <linux/export.h>
 
 7#include <linux/string.h>
 8#include <linux/types.h>
 9
10#include <asm/mce.h>
11
12#ifdef CONFIG_X86_MCE
13static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
14
15void enable_copy_mc_fragile(void)
16{
17	static_branch_inc(&copy_mc_fragile_key);
18}
19#define copy_mc_fragile_enabled (static_branch_unlikely(&copy_mc_fragile_key))
20
21/*
22 * Similar to copy_user_handle_tail, probe for the write fault point, or
23 * source exception point.
24 */
25__visible notrace unsigned long
26copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
27{
28	for (; len; --len, to++, from++)
29		if (copy_mc_fragile(to, from, 1))
30			break;
31	return len;
32}
33#else
34/*
35 * No point in doing careful copying, or consulting a static key when
36 * there is no #MC handler in the CONFIG_X86_MCE=n case.
37 */
38void enable_copy_mc_fragile(void)
39{
40}
41#define copy_mc_fragile_enabled (0)
42#endif
43
44unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
45
46/**
47 * copy_mc_to_kernel - memory copy that handles source exceptions
48 *
49 * @dst:	destination address
50 * @src:	source address
51 * @len:	number of bytes to copy
52 *
53 * Call into the 'fragile' version on systems that benefit from avoiding
54 * corner case poison consumption scenarios, For example, accessing
55 * poison across 2 cachelines with a single instruction. Almost all
56 * other uses case can use copy_mc_enhanced_fast_string() for a fast
57 * recoverable copy, or fallback to plain memcpy.
58 *
59 * Return 0 for success, or number of bytes not copied if there was an
60 * exception.
61 */
62unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
63{
64	if (copy_mc_fragile_enabled)
65		return copy_mc_fragile(dst, src, len);
66	if (static_cpu_has(X86_FEATURE_ERMS))
67		return copy_mc_enhanced_fast_string(dst, src, len);
 
 
 
 
 
 
 
 
 
 
68	memcpy(dst, src, len);
69	return 0;
70}
71EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
72
73unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, unsigned len)
74{
75	unsigned long ret;
76
77	if (copy_mc_fragile_enabled) {
 
78		__uaccess_begin();
79		ret = copy_mc_fragile((__force void *)dst, src, len);
80		__uaccess_end();
81		return ret;
82	}
83
84	if (static_cpu_has(X86_FEATURE_ERMS)) {
 
85		__uaccess_begin();
86		ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len);
87		__uaccess_end();
88		return ret;
89	}
90
91	return copy_user_generic((__force void *)dst, src, len);
92}

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
  3
  4#include <linux/jump_label.h>
  5#include <linux/uaccess.h>
  6#include <linux/export.h>
  7#include <linux/instrumented.h>
  8#include <linux/string.h>
  9#include <linux/types.h>
 10
 11#include <asm/mce.h>
 12
 13#ifdef CONFIG_X86_MCE
 14static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
 15
 16void enable_copy_mc_fragile(void)
 17{
 18	static_branch_inc(&copy_mc_fragile_key);
 19}
 20#define copy_mc_fragile_enabled (static_branch_unlikely(&copy_mc_fragile_key))
 21
 22/*
 23 * Similar to copy_user_handle_tail, probe for the write fault point, or
 24 * source exception point.
 25 */
 26__visible notrace unsigned long
 27copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
 28{
 29	for (; len; --len, to++, from++)
 30		if (copy_mc_fragile(to, from, 1))
 31			break;
 32	return len;
 33}
 34#else
 35/*
 36 * No point in doing careful copying, or consulting a static key when
 37 * there is no #MC handler in the CONFIG_X86_MCE=n case.
 38 */
 39void enable_copy_mc_fragile(void)
 40{
 41}
 42#define copy_mc_fragile_enabled (0)
 43#endif
 44
 45unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
 46
 47/**
 48 * copy_mc_to_kernel - memory copy that handles source exceptions
 49 *
 50 * @dst:	destination address
 51 * @src:	source address
 52 * @len:	number of bytes to copy
 53 *
 54 * Call into the 'fragile' version on systems that benefit from avoiding
 55 * corner case poison consumption scenarios, For example, accessing
 56 * poison across 2 cachelines with a single instruction. Almost all
 57 * other uses case can use copy_mc_enhanced_fast_string() for a fast
 58 * recoverable copy, or fallback to plain memcpy.
 59 *
 60 * Return 0 for success, or number of bytes not copied if there was an
 61 * exception.
 62 */
 63unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
 64{
 65	unsigned long ret;
 66
 67	if (copy_mc_fragile_enabled) {
 68		instrument_memcpy_before(dst, src, len);
 69		ret = copy_mc_fragile(dst, src, len);
 70		instrument_memcpy_after(dst, src, len, ret);
 71		return ret;
 72	}
 73	if (static_cpu_has(X86_FEATURE_ERMS)) {
 74		instrument_memcpy_before(dst, src, len);
 75		ret = copy_mc_enhanced_fast_string(dst, src, len);
 76		instrument_memcpy_after(dst, src, len, ret);
 77		return ret;
 78	}
 79	memcpy(dst, src, len);
 80	return 0;
 81}
 82EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
 83
 84unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, unsigned len)
 85{
 86	unsigned long ret;
 87
 88	if (copy_mc_fragile_enabled) {
 89		instrument_copy_to_user(dst, src, len);
 90		__uaccess_begin();
 91		ret = copy_mc_fragile((__force void *)dst, src, len);
 92		__uaccess_end();
 93		return ret;
 94	}
 95
 96	if (static_cpu_has(X86_FEATURE_ERMS)) {
 97		instrument_copy_to_user(dst, src, len);
 98		__uaccess_begin();
 99		ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len);
100		__uaccess_end();
101		return ret;
102	}
103
104	return copy_user_generic((__force void *)dst, src, len);
105}