1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/mm.h>
  3#include <linux/gfp.h>
  4#include <linux/kernel.h>
  5
  6#include <asm/mce.h>
  7
  8#include "debugfs.h"
  9
 10/*
 11 * RAS Correctable Errors Collector
 12 *
 13 * This is a simple gadget which collects correctable errors and counts their
 14 * occurrence per physical page address.
 15 *
 16 * We've opted for possibly the simplest data structure to collect those - an
 17 * array of the size of a memory page. It stores 512 u64's with the following
 18 * structure:
 19 *
 20 * [63 ... PFN ... 12 | 11 ... generation ... 10 | 9 ... count ... 0]
 21 *
 22 * The generation in the two highest order bits is two bits which are set to 11b
 23 * on every insertion. During the course of each entry's existence, the
 24 * generation field gets decremented during spring cleaning to 10b, then 01b and
 25 * then 00b.
 26 *
 27 * This way we're employing the natural numeric ordering to make sure that newly
 28 * inserted/touched elements have higher 12-bit counts (which we've manufactured)
 29 * and thus iterating over the array initially won't kick out those elements
 30 * which were inserted last.
 31 *
 32 * Spring cleaning is what we do when we reach a certain number CLEAN_ELEMS of
 33 * elements entered into the array, during which, we're decaying all elements.
 34 * If, after decay, an element gets inserted again, its generation is set to 11b
 35 * to make sure it has higher numerical count than other, older elements and
 36 * thus emulate an an LRU-like behavior when deleting elements to free up space
 37 * in the page.
 38 *
 39 * When an element reaches it's max count of count_threshold, we try to poison
 40 * it by assuming that errors triggered count_threshold times in a single page
 41 * are excessive and that page shouldn't be used anymore. count_threshold is
 42 * initialized to COUNT_MASK which is the maximum.
 43 *
 44 * That error event entry causes cec_add_elem() to return !0 value and thus
 45 * signal to its callers to log the error.
 46 *
 47 * To the question why we've chosen a page and moving elements around with
 48 * memmove(), it is because it is a very simple structure to handle and max data
 49 * movement is 4K which on highly optimized modern CPUs is almost unnoticeable.
 50 * We wanted to avoid the pointer traversal of more complex structures like a
 51 * linked list or some sort of a balancing search tree.
 52 *
 53 * Deleting an element takes O(n) but since it is only a single page, it should
 54 * be fast enough and it shouldn't happen all too often depending on error
 55 * patterns.
 56 */
 57
 58#undef pr_fmt
 59#define pr_fmt(fmt) "RAS: " fmt
 60
 61/*
 62 * We use DECAY_BITS bits of PAGE_SHIFT bits for counting decay, i.e., how long
 63 * elements have stayed in the array without having been accessed again.
 64 */
 65#define DECAY_BITS		2
 66#define DECAY_MASK		((1ULL << DECAY_BITS) - 1)
 67#define MAX_ELEMS		(PAGE_SIZE / sizeof(u64))
 68
 69/*
 70 * Threshold amount of inserted elements after which we start spring
 71 * cleaning.
 72 */
 73#define CLEAN_ELEMS		(MAX_ELEMS >> DECAY_BITS)
 74
 75/* Bits which count the number of errors happened in this 4K page. */
 76#define COUNT_BITS		(PAGE_SHIFT - DECAY_BITS)
 77#define COUNT_MASK		((1ULL << COUNT_BITS) - 1)
 78#define FULL_COUNT_MASK		(PAGE_SIZE - 1)
 79
 80/*
 81 * u64: [ 63 ... 12 | DECAY_BITS | COUNT_BITS ]
 82 */
 83
 84#define PFN(e)			((e) >> PAGE_SHIFT)
 85#define DECAY(e)		(((e) >> COUNT_BITS) & DECAY_MASK)
 86#define COUNT(e)		((unsigned int)(e) & COUNT_MASK)
 87#define FULL_COUNT(e)		((e) & (PAGE_SIZE - 1))
 88
 89static struct ce_array {
 90	u64 *array;			/* container page */
 91	unsigned int n;			/* number of elements in the array */
 92
 93	unsigned int decay_count;	/*
 94					 * number of element insertions/increments
 95					 * since the last spring cleaning.
 96					 */
 97
 98	u64 pfns_poisoned;		/*
 99					 * number of PFNs which got poisoned.
100					 */
101
102	u64 ces_entered;		/*
103					 * The number of correctable errors
104					 * entered into the collector.
105					 */
106
107	u64 decays_done;		/*
108					 * Times we did spring cleaning.
109					 */
110
111	union {
112		struct {
113			__u32	disabled : 1,	/* cmdline disabled */
114			__resv   : 31;
115		};
116		__u32 flags;
117	};
118} ce_arr;
119
120static DEFINE_MUTEX(ce_mutex);
121static u64 dfs_pfn;
122
123/* Amount of errors after which we offline */
124static unsigned int count_threshold = COUNT_MASK;
125
126/*
127 * The timer "decays" element count each timer_interval which is 24hrs by
128 * default.
129 */
130
131#define CEC_TIMER_DEFAULT_INTERVAL	24 * 60 * 60	/* 24 hrs */
132#define CEC_TIMER_MIN_INTERVAL		 1 * 60 * 60	/* 1h */
133#define CEC_TIMER_MAX_INTERVAL	   30 *	24 * 60 * 60	/* one month */
134static struct timer_list cec_timer;
135static u64 timer_interval = CEC_TIMER_DEFAULT_INTERVAL;
136
137/*
138 * Decrement decay value. We're using DECAY_BITS bits to denote decay of an
139 * element in the array. On insertion and any access, it gets reset to max.
140 */
141static void do_spring_cleaning(struct ce_array *ca)
142{
143	int i;
144
145	for (i = 0; i < ca->n; i++) {
146		u8 decay = DECAY(ca->array[i]);
147
148		if (!decay)
149			continue;
150
151		decay--;
152
153		ca->array[i] &= ~(DECAY_MASK << COUNT_BITS);
154		ca->array[i] |= (decay << COUNT_BITS);
155	}
156	ca->decay_count = 0;
157	ca->decays_done++;
158}
159
160/*
161 * @interval in seconds
162 */
163static void cec_mod_timer(struct timer_list *t, unsigned long interval)
164{
165	unsigned long iv;
166
167	iv = interval * HZ + jiffies;
168
169	mod_timer(t, round_jiffies(iv));
170}
171
172static void cec_timer_fn(struct timer_list *unused)
173{
174	do_spring_cleaning(&ce_arr);
175
176	cec_mod_timer(&cec_timer, timer_interval);
177}
178
179/*
180 * @to: index of the smallest element which is >= then @pfn.
181 *
182 * Return the index of the pfn if found, otherwise negative value.
183 */
184static int __find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)
185{
186	u64 this_pfn;
187	int min = 0, max = ca->n;
188
189	while (min < max) {
190		int tmp = (max + min) >> 1;
191
192		this_pfn = PFN(ca->array[tmp]);
193
194		if (this_pfn < pfn)
195			min = tmp + 1;
196		else if (this_pfn > pfn)
197			max = tmp;
198		else {
199			min = tmp;
200			break;
201		}
202	}
203
204	if (to)
205		*to = min;
206
207	this_pfn = PFN(ca->array[min]);
208
209	if (this_pfn == pfn)
210		return min;
211
212	return -ENOKEY;
213}
214
215static int find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)
216{
217	WARN_ON(!to);
218
219	if (!ca->n) {
220		*to = 0;
221		return -ENOKEY;
222	}
223	return __find_elem(ca, pfn, to);
224}
225
226static void del_elem(struct ce_array *ca, int idx)
227{
228	/* Save us a function call when deleting the last element. */
229	if (ca->n - (idx + 1))
230		memmove((void *)&ca->array[idx],
231			(void *)&ca->array[idx + 1],
232			(ca->n - (idx + 1)) * sizeof(u64));
233
234	ca->n--;
235}
236
237static u64 del_lru_elem_unlocked(struct ce_array *ca)
238{
239	unsigned int min = FULL_COUNT_MASK;
240	int i, min_idx = 0;
241
242	for (i = 0; i < ca->n; i++) {
243		unsigned int this = FULL_COUNT(ca->array[i]);
244
245		if (min > this) {
246			min = this;
247			min_idx = i;
248		}
249	}
250
251	del_elem(ca, min_idx);
252
253	return PFN(ca->array[min_idx]);
254}
255
256/*
257 * We return the 0th pfn in the error case under the assumption that it cannot
258 * be poisoned and excessive CEs in there are a serious deal anyway.
259 */
260static u64 __maybe_unused del_lru_elem(void)
261{
262	struct ce_array *ca = &ce_arr;
263	u64 pfn;
264
265	if (!ca->n)
266		return 0;
267
268	mutex_lock(&ce_mutex);
269	pfn = del_lru_elem_unlocked(ca);
270	mutex_unlock(&ce_mutex);
271
272	return pfn;
273}
274
275
276int cec_add_elem(u64 pfn)
277{
278	struct ce_array *ca = &ce_arr;
279	unsigned int to;
280	int count, ret = 0;
281
282	/*
283	 * We can be called very early on the identify_cpu() path where we are
284	 * not initialized yet. We ignore the error for simplicity.
285	 */
286	if (!ce_arr.array || ce_arr.disabled)
287		return -ENODEV;
288
289	ca->ces_entered++;
290
291	mutex_lock(&ce_mutex);
292
293	if (ca->n == MAX_ELEMS)
294		WARN_ON(!del_lru_elem_unlocked(ca));
295
296	ret = find_elem(ca, pfn, &to);
297	if (ret < 0) {
298		/*
299		 * Shift range [to-end] to make room for one more element.
300		 */
301		memmove((void *)&ca->array[to + 1],
302			(void *)&ca->array[to],
303			(ca->n - to) * sizeof(u64));
304
305		ca->array[to] = (pfn << PAGE_SHIFT) |
306				(DECAY_MASK << COUNT_BITS) | 1;
307
308		ca->n++;
309
310		ret = 0;
311
312		goto decay;
313	}
314
315	count = COUNT(ca->array[to]);
316
317	if (count < count_threshold) {
318		ca->array[to] |= (DECAY_MASK << COUNT_BITS);
319		ca->array[to]++;
320
321		ret = 0;
322	} else {
323		u64 pfn = ca->array[to] >> PAGE_SHIFT;
324
325		if (!pfn_valid(pfn)) {
326			pr_warn("CEC: Invalid pfn: 0x%llx\n", pfn);
327		} else {
328			/* We have reached max count for this page, soft-offline it. */
329			pr_err("Soft-offlining pfn: 0x%llx\n", pfn);
330			memory_failure_queue(pfn, MF_SOFT_OFFLINE);
331			ca->pfns_poisoned++;
332		}
333
334		del_elem(ca, to);
335
336		/*
337		 * Return a >0 value to denote that we've reached the offlining
338		 * threshold.
339		 */
340		ret = 1;
341
342		goto unlock;
343	}
344
345decay:
346	ca->decay_count++;
347
348	if (ca->decay_count >= CLEAN_ELEMS)
349		do_spring_cleaning(ca);
350
351unlock:
352	mutex_unlock(&ce_mutex);
353
354	return ret;
355}
356
357static int u64_get(void *data, u64 *val)
358{
359	*val = *(u64 *)data;
360
361	return 0;
362}
363
364static int pfn_set(void *data, u64 val)
365{
366	*(u64 *)data = val;
367
368	return cec_add_elem(val);
369}
370
371DEFINE_DEBUGFS_ATTRIBUTE(pfn_ops, u64_get, pfn_set, "0x%llx\n");
372
373static int decay_interval_set(void *data, u64 val)
374{
375	*(u64 *)data = val;
376
377	if (val < CEC_TIMER_MIN_INTERVAL)
378		return -EINVAL;
379
380	if (val > CEC_TIMER_MAX_INTERVAL)
381		return -EINVAL;
382
383	timer_interval = val;
384
385	cec_mod_timer(&cec_timer, timer_interval);
386	return 0;
387}
388DEFINE_DEBUGFS_ATTRIBUTE(decay_interval_ops, u64_get, decay_interval_set, "%lld\n");
389
390static int count_threshold_set(void *data, u64 val)
391{
392	*(u64 *)data = val;
393
394	if (val > COUNT_MASK)
395		val = COUNT_MASK;
396
397	count_threshold = val;
398
399	return 0;
400}
401DEFINE_DEBUGFS_ATTRIBUTE(count_threshold_ops, u64_get, count_threshold_set, "%lld\n");
402
403static int array_dump(struct seq_file *m, void *v)
404{
405	struct ce_array *ca = &ce_arr;
406	u64 prev = 0;
407	int i;
408
409	mutex_lock(&ce_mutex);
410
411	seq_printf(m, "{ n: %d\n", ca->n);
412	for (i = 0; i < ca->n; i++) {
413		u64 this = PFN(ca->array[i]);
414
415		seq_printf(m, " %03d: [%016llx|%03llx]\n", i, this, FULL_COUNT(ca->array[i]));
416
417		WARN_ON(prev > this);
418
419		prev = this;
420	}
421
422	seq_printf(m, "}\n");
423
424	seq_printf(m, "Stats:\nCEs: %llu\nofflined pages: %llu\n",
425		   ca->ces_entered, ca->pfns_poisoned);
426
427	seq_printf(m, "Flags: 0x%x\n", ca->flags);
428
429	seq_printf(m, "Timer interval: %lld seconds\n", timer_interval);
430	seq_printf(m, "Decays: %lld\n", ca->decays_done);
431
432	seq_printf(m, "Action threshold: %d\n", count_threshold);
433
434	mutex_unlock(&ce_mutex);
435
436	return 0;
437}
438
439static int array_open(struct inode *inode, struct file *filp)
440{
441	return single_open(filp, array_dump, NULL);
442}
443
444static const struct file_operations array_ops = {
445	.owner	 = THIS_MODULE,
446	.open	 = array_open,
447	.read	 = seq_read,
448	.llseek	 = seq_lseek,
449	.release = single_release,
450};
451
452static int __init create_debugfs_nodes(void)
453{
454	struct dentry *d, *pfn, *decay, *count, *array;
455
456	d = debugfs_create_dir("cec", ras_debugfs_dir);
457	if (!d) {
458		pr_warn("Error creating cec debugfs node!\n");
459		return -1;
460	}
461
462	pfn = debugfs_create_file("pfn", S_IRUSR | S_IWUSR, d, &dfs_pfn, &pfn_ops);
463	if (!pfn) {
464		pr_warn("Error creating pfn debugfs node!\n");
465		goto err;
466	}
467
468	array = debugfs_create_file("array", S_IRUSR, d, NULL, &array_ops);
469	if (!array) {
470		pr_warn("Error creating array debugfs node!\n");
471		goto err;
472	}
473
474	decay = debugfs_create_file("decay_interval", S_IRUSR | S_IWUSR, d,
475				    &timer_interval, &decay_interval_ops);
476	if (!decay) {
477		pr_warn("Error creating decay_interval debugfs node!\n");
478		goto err;
479	}
480
481	count = debugfs_create_file("count_threshold", S_IRUSR | S_IWUSR, d,
482				    &count_threshold, &count_threshold_ops);
483	if (!count) {
484		pr_warn("Error creating count_threshold debugfs node!\n");
485		goto err;
486	}
487
488
489	return 0;
490
491err:
492	debugfs_remove_recursive(d);
493
494	return 1;
495}
496
497void __init cec_init(void)
498{
499	if (ce_arr.disabled)
500		return;
501
502	ce_arr.array = (void *)get_zeroed_page(GFP_KERNEL);
503	if (!ce_arr.array) {
504		pr_err("Error allocating CE array page!\n");
505		return;
506	}
507
508	if (create_debugfs_nodes())
509		return;
510
511	timer_setup(&cec_timer, cec_timer_fn, 0);
512	cec_mod_timer(&cec_timer, CEC_TIMER_DEFAULT_INTERVAL);
513
514	pr_info("Correctable Errors collector initialized.\n");
515}
516
517int __init parse_cec_param(char *str)
518{
519	if (!str)
520		return 0;
521
522	if (*str == '=')
523		str++;
524
525	if (!strcmp(str, "cec_disable"))
526		ce_arr.disabled = 1;
527	else
528		return 0;
529
530	return 1;
531}