1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Performance events ring-buffer code:
  4 *
  5 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
  6 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
  7 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
  8 *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 
 
  9 */
 10
 11#include <linux/perf_event.h>
 12#include <linux/vmalloc.h>
 13#include <linux/slab.h>
 14#include <linux/circ_buf.h>
 15#include <linux/poll.h>
 16#include <linux/nospec.h>
 17
 18#include "internal.h"
 19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 20static void perf_output_wakeup(struct perf_output_handle *handle)
 21{
 22	atomic_set(&handle->rb->poll, EPOLLIN);
 23
 24	handle->event->pending_wakeup = 1;
 25	irq_work_queue(&handle->event->pending_irq);
 26}
 27
 28/*
 29 * We need to ensure a later event_id doesn't publish a head when a former
 30 * event isn't done writing. However since we need to deal with NMIs we
 31 * cannot fully serialize things.
 32 *
 33 * We only publish the head (and generate a wakeup) when the outer-most
 34 * event completes.
 35 */
 36static void perf_output_get_handle(struct perf_output_handle *handle)
 37{
 38	struct perf_buffer *rb = handle->rb;
 39
 40	preempt_disable();
 41
 42	/*
 43	 * Avoid an explicit LOAD/STORE such that architectures with memops
 44	 * can use them.
 45	 */
 46	(*(volatile unsigned int *)&rb->nest)++;
 47	handle->wakeup = local_read(&rb->wakeup);
 48}
 49
 50static void perf_output_put_handle(struct perf_output_handle *handle)
 51{
 52	struct perf_buffer *rb = handle->rb;
 53	unsigned long head;
 54	unsigned int nest;
 55
 56	/*
 57	 * If this isn't the outermost nesting, we don't have to update
 58	 * @rb->user_page->data_head.
 59	 */
 60	nest = READ_ONCE(rb->nest);
 61	if (nest > 1) {
 62		WRITE_ONCE(rb->nest, nest - 1);
 63		goto out;
 64	}
 65
 66again:
 67	/*
 68	 * In order to avoid publishing a head value that goes backwards,
 69	 * we must ensure the load of @rb->head happens after we've
 70	 * incremented @rb->nest.
 71	 *
 72	 * Otherwise we can observe a @rb->head value before one published
 73	 * by an IRQ/NMI happening between the load and the increment.
 74	 */
 75	barrier();
 76	head = local_read(&rb->head);
 77
 78	/*
 79	 * IRQ/NMI can happen here and advance @rb->head, causing our
 80	 * load above to be stale.
 81	 */
 82
 83	/*
 84	 * Since the mmap() consumer (userspace) can run on a different CPU:
 85	 *
 86	 *   kernel				user
 87	 *
 88	 *   if (LOAD ->data_tail) {		LOAD ->data_head
 89	 *			(A)		smp_rmb()	(C)
 90	 *	STORE $data			LOAD $data
 91	 *	smp_wmb()	(B)		smp_mb()	(D)
 92	 *	STORE ->data_head		STORE ->data_tail
 93	 *   }
 94	 *
 95	 * Where A pairs with D, and B pairs with C.
 96	 *
 97	 * In our case (A) is a control dependency that separates the load of
 98	 * the ->data_tail and the stores of $data. In case ->data_tail
 99	 * indicates there is no room in the buffer to store $data we do not.
100	 *
101	 * D needs to be a full barrier since it separates the data READ
102	 * from the tail WRITE.
103	 *
104	 * For B a WMB is sufficient since it separates two WRITEs, and for C
105	 * an RMB is sufficient since it separates two READs.
106	 *
107	 * See perf_output_begin().
108	 */
109	smp_wmb(); /* B, matches C */
110	WRITE_ONCE(rb->user_page->data_head, head);
111
112	/*
113	 * We must publish the head before decrementing the nest count,
114	 * otherwise an IRQ/NMI can publish a more recent head value and our
115	 * write will (temporarily) publish a stale value.
116	 */
117	barrier();
118	WRITE_ONCE(rb->nest, 0);
119
120	/*
121	 * Ensure we decrement @rb->nest before we validate the @rb->head.
122	 * Otherwise we cannot be sure we caught the 'last' nested update.
123	 */
124	barrier();
125	if (unlikely(head != local_read(&rb->head))) {
126		WRITE_ONCE(rb->nest, 1);
127		goto again;
128	}
129
130	if (handle->wakeup != local_read(&rb->wakeup))
131		perf_output_wakeup(handle);
132
133out:
134	preempt_enable();
135}
136
137static __always_inline bool
138ring_buffer_has_space(unsigned long head, unsigned long tail,
139		      unsigned long data_size, unsigned int size,
140		      bool backward)
141{
142	if (!backward)
143		return CIRC_SPACE(head, tail, data_size) >= size;
144	else
145		return CIRC_SPACE(tail, head, data_size) >= size;
146}
147
148static __always_inline int
149__perf_output_begin(struct perf_output_handle *handle,
150		    struct perf_sample_data *data,
151		    struct perf_event *event, unsigned int size,
152		    bool backward)
153{
154	struct perf_buffer *rb;
155	unsigned long tail, offset, head;
156	int have_lost, page_shift;
 
157	struct {
158		struct perf_event_header header;
159		u64			 id;
160		u64			 lost;
161	} lost_event;
162
163	rcu_read_lock();
164	/*
165	 * For inherited events we send all the output towards the parent.
166	 */
167	if (event->parent)
168		event = event->parent;
169
170	rb = rcu_dereference(event->rb);
171	if (unlikely(!rb))
172		goto out;
173
174	if (unlikely(rb->paused)) {
175		if (rb->nr_pages) {
176			local_inc(&rb->lost);
177			atomic64_inc(&event->lost_samples);
178		}
179		goto out;
180	}
181
182	handle->rb    = rb;
183	handle->event = event;
184
185	have_lost = local_read(&rb->lost);
186	if (unlikely(have_lost)) {
187		size += sizeof(lost_event);
188		if (event->attr.sample_id_all)
189			size += event->id_header_size;
 
190	}
191
192	perf_output_get_handle(handle);
193
194	do {
195		tail = READ_ONCE(rb->user_page->data_tail);
196		offset = head = local_read(&rb->head);
197		if (!rb->overwrite) {
198			if (unlikely(!ring_buffer_has_space(head, tail,
199							    perf_data_size(rb),
200							    size, backward)))
201				goto fail;
202		}
203
204		/*
205		 * The above forms a control dependency barrier separating the
206		 * @tail load above from the data stores below. Since the @tail
207		 * load is required to compute the branch to fail below.
208		 *
209		 * A, matches D; the full memory barrier userspace SHOULD issue
210		 * after reading the data and before storing the new tail
211		 * position.
212		 *
213		 * See perf_output_put_handle().
214		 */
215
216		if (!backward)
217			head += size;
218		else
219			head -= size;
 
220	} while (local_cmpxchg(&rb->head, offset, head) != offset);
221
222	if (backward) {
223		offset = head;
224		head = (u64)(-head);
225	}
226
227	/*
228	 * We rely on the implied barrier() by local_cmpxchg() to ensure
229	 * none of the data stores below can be lifted up by the compiler.
230	 */
231
232	if (unlikely(head - local_read(&rb->wakeup) > rb->watermark))
233		local_add(rb->watermark, &rb->wakeup);
234
235	page_shift = PAGE_SHIFT + page_order(rb);
 
 
 
 
 
236
237	handle->page = (offset >> page_shift) & (rb->nr_pages - 1);
238	offset &= (1UL << page_shift) - 1;
239	handle->addr = rb->data_pages[handle->page] + offset;
240	handle->size = (1UL << page_shift) - offset;
241
242	if (unlikely(have_lost)) {
243		lost_event.header.size = sizeof(lost_event);
244		lost_event.header.type = PERF_RECORD_LOST;
245		lost_event.header.misc = 0;
246		lost_event.id          = event->id;
247		lost_event.lost        = local_xchg(&rb->lost, 0);
248
249		/* XXX mostly redundant; @data is already fully initializes */
250		perf_event_header__init_id(&lost_event.header, data, event);
251		perf_output_put(handle, lost_event);
252		perf_event__output_id_sample(event, handle, data);
253	}
254
255	return 0;
256
257fail:
258	local_inc(&rb->lost);
259	atomic64_inc(&event->lost_samples);
260	perf_output_put_handle(handle);
261out:
262	rcu_read_unlock();
263
264	return -ENOSPC;
265}
266
267int perf_output_begin_forward(struct perf_output_handle *handle,
268			      struct perf_sample_data *data,
269			      struct perf_event *event, unsigned int size)
270{
271	return __perf_output_begin(handle, data, event, size, false);
272}
273
274int perf_output_begin_backward(struct perf_output_handle *handle,
275			       struct perf_sample_data *data,
276			       struct perf_event *event, unsigned int size)
277{
278	return __perf_output_begin(handle, data, event, size, true);
279}
280
281int perf_output_begin(struct perf_output_handle *handle,
282		      struct perf_sample_data *data,
283		      struct perf_event *event, unsigned int size)
284{
285
286	return __perf_output_begin(handle, data, event, size,
287				   unlikely(is_write_backward(event)));
288}
289
290unsigned int perf_output_copy(struct perf_output_handle *handle,
291		      const void *buf, unsigned int len)
292{
293	return __output_copy(handle, buf, len);
294}
295
296unsigned int perf_output_skip(struct perf_output_handle *handle,
297			      unsigned int len)
298{
299	return __output_skip(handle, NULL, len);
300}
301
302void perf_output_end(struct perf_output_handle *handle)
303{
304	perf_output_put_handle(handle);
305	rcu_read_unlock();
306}
307
308static void
309ring_buffer_init(struct perf_buffer *rb, long watermark, int flags)
310{
311	long max_size = perf_data_size(rb);
312
313	if (watermark)
314		rb->watermark = min(max_size, watermark);
315
316	if (!rb->watermark)
317		rb->watermark = max_size / 2;
318
319	if (flags & RING_BUFFER_WRITABLE)
320		rb->overwrite = 0;
321	else
322		rb->overwrite = 1;
323
324	refcount_set(&rb->refcount, 1);
325
326	INIT_LIST_HEAD(&rb->event_list);
327	spin_lock_init(&rb->event_lock);
328
329	/*
330	 * perf_output_begin() only checks rb->paused, therefore
331	 * rb->paused must be true if we have no pages for output.
332	 */
333	if (!rb->nr_pages)
334		rb->paused = 1;
335}
336
337void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags)
338{
339	/*
340	 * OVERWRITE is determined by perf_aux_output_end() and can't
341	 * be passed in directly.
342	 */
343	if (WARN_ON_ONCE(flags & PERF_AUX_FLAG_OVERWRITE))
344		return;
345
346	handle->aux_flags |= flags;
347}
348EXPORT_SYMBOL_GPL(perf_aux_output_flag);
349
350/*
351 * This is called before hardware starts writing to the AUX area to
352 * obtain an output handle and make sure there's room in the buffer.
353 * When the capture completes, call perf_aux_output_end() to commit
354 * the recorded data to the buffer.
355 *
356 * The ordering is similar to that of perf_output_{begin,end}, with
357 * the exception of (B), which should be taken care of by the pmu
358 * driver, since ordering rules will differ depending on hardware.
359 *
360 * Call this from pmu::start(); see the comment in perf_aux_output_end()
361 * about its use in pmu callbacks. Both can also be called from the PMI
362 * handler if needed.
363 */
364void *perf_aux_output_begin(struct perf_output_handle *handle,
365			    struct perf_event *event)
366{
367	struct perf_event *output_event = event;
368	unsigned long aux_head, aux_tail;
369	struct perf_buffer *rb;
370	unsigned int nest;
371
372	if (output_event->parent)
373		output_event = output_event->parent;
374
375	/*
376	 * Since this will typically be open across pmu::add/pmu::del, we
377	 * grab ring_buffer's refcount instead of holding rcu read lock
378	 * to make sure it doesn't disappear under us.
379	 */
380	rb = ring_buffer_get(output_event);
381	if (!rb)
382		return NULL;
383
384	if (!rb_has_aux(rb))
385		goto err;
386
387	/*
388	 * If aux_mmap_count is zero, the aux buffer is in perf_mmap_close(),
389	 * about to get freed, so we leave immediately.
390	 *
391	 * Checking rb::aux_mmap_count and rb::refcount has to be done in
392	 * the same order, see perf_mmap_close. Otherwise we end up freeing
393	 * aux pages in this path, which is a bug, because in_atomic().
394	 */
395	if (!atomic_read(&rb->aux_mmap_count))
396		goto err;
397
398	if (!refcount_inc_not_zero(&rb->aux_refcount))
399		goto err;
400
401	nest = READ_ONCE(rb->aux_nest);
402	/*
403	 * Nesting is not supported for AUX area, make sure nested
404	 * writers are caught early
405	 */
406	if (WARN_ON_ONCE(nest))
407		goto err_put;
408
409	WRITE_ONCE(rb->aux_nest, nest + 1);
410
411	aux_head = rb->aux_head;
412
413	handle->rb = rb;
414	handle->event = event;
415	handle->head = aux_head;
416	handle->size = 0;
417	handle->aux_flags = 0;
418
419	/*
420	 * In overwrite mode, AUX data stores do not depend on aux_tail,
421	 * therefore (A) control dependency barrier does not exist. The
422	 * (B) <-> (C) ordering is still observed by the pmu driver.
423	 */
424	if (!rb->aux_overwrite) {
425		aux_tail = READ_ONCE(rb->user_page->aux_tail);
426		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;
427		if (aux_head - aux_tail < perf_aux_size(rb))
428			handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));
429
430		/*
431		 * handle->size computation depends on aux_tail load; this forms a
432		 * control dependency barrier separating aux_tail load from aux data
433		 * store that will be enabled on successful return
434		 */
435		if (!handle->size) { /* A, matches D */
436			event->pending_disable = smp_processor_id();
437			perf_output_wakeup(handle);
438			WRITE_ONCE(rb->aux_nest, 0);
439			goto err_put;
440		}
441	}
442
443	return handle->rb->aux_priv;
444
445err_put:
446	/* can't be last */
447	rb_free_aux(rb);
448
449err:
450	ring_buffer_put(rb);
451	handle->event = NULL;
452
453	return NULL;
454}
455EXPORT_SYMBOL_GPL(perf_aux_output_begin);
456
457static __always_inline bool rb_need_aux_wakeup(struct perf_buffer *rb)
458{
459	if (rb->aux_overwrite)
460		return false;
461
462	if (rb->aux_head - rb->aux_wakeup >= rb->aux_watermark) {
463		rb->aux_wakeup = rounddown(rb->aux_head, rb->aux_watermark);
464		return true;
465	}
466
467	return false;
468}
469
470/*
471 * Commit the data written by hardware into the ring buffer by adjusting
472 * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the
473 * pmu driver's responsibility to observe ordering rules of the hardware,
474 * so that all the data is externally visible before this is called.
475 *
476 * Note: this has to be called from pmu::stop() callback, as the assumption
477 * of the AUX buffer management code is that after pmu::stop(), the AUX
478 * transaction must be stopped and therefore drop the AUX reference count.
479 */
480void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)
481{
482	bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED);
483	struct perf_buffer *rb = handle->rb;
484	unsigned long aux_head;
485
486	/* in overwrite mode, driver provides aux_head via handle */
487	if (rb->aux_overwrite) {
488		handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE;
489
490		aux_head = handle->head;
491		rb->aux_head = aux_head;
492	} else {
493		handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE;
494
495		aux_head = rb->aux_head;
496		rb->aux_head += size;
497	}
498
499	/*
500	 * Only send RECORD_AUX if we have something useful to communicate
501	 *
502	 * Note: the OVERWRITE records by themselves are not considered
503	 * useful, as they don't communicate any *new* information,
504	 * aside from the short-lived offset, that becomes history at
505	 * the next event sched-in and therefore isn't useful.
506	 * The userspace that needs to copy out AUX data in overwrite
507	 * mode should know to use user_page::aux_head for the actual
508	 * offset. So, from now on we don't output AUX records that
509	 * have *only* OVERWRITE flag set.
510	 */
511	if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE))
512		perf_event_aux_event(handle->event, aux_head, size,
513				     handle->aux_flags);
514
515	WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);
516	if (rb_need_aux_wakeup(rb))
517		wakeup = true;
518
519	if (wakeup) {
520		if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)
521			handle->event->pending_disable = smp_processor_id();
522		perf_output_wakeup(handle);
523	}
524
525	handle->event = NULL;
526
527	WRITE_ONCE(rb->aux_nest, 0);
528	/* can't be last */
529	rb_free_aux(rb);
530	ring_buffer_put(rb);
531}
532EXPORT_SYMBOL_GPL(perf_aux_output_end);
533
534/*
535 * Skip over a given number of bytes in the AUX buffer, due to, for example,
536 * hardware's alignment constraints.
537 */
538int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)
539{
540	struct perf_buffer *rb = handle->rb;
541
542	if (size > handle->size)
543		return -ENOSPC;
544
545	rb->aux_head += size;
546
547	WRITE_ONCE(rb->user_page->aux_head, rb->aux_head);
548	if (rb_need_aux_wakeup(rb)) {
549		perf_output_wakeup(handle);
550		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;
551	}
552
553	handle->head = rb->aux_head;
554	handle->size -= size;
555
556	return 0;
557}
558EXPORT_SYMBOL_GPL(perf_aux_output_skip);
559
560void *perf_get_aux(struct perf_output_handle *handle)
561{
562	/* this is only valid between perf_aux_output_begin and *_end */
563	if (!handle->event)
564		return NULL;
565
566	return handle->rb->aux_priv;
567}
568EXPORT_SYMBOL_GPL(perf_get_aux);
569
570/*
571 * Copy out AUX data from an AUX handle.
572 */
573long perf_output_copy_aux(struct perf_output_handle *aux_handle,
574			  struct perf_output_handle *handle,
575			  unsigned long from, unsigned long to)
576{
577	struct perf_buffer *rb = aux_handle->rb;
578	unsigned long tocopy, remainder, len = 0;
579	void *addr;
580
581	from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
582	to &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
583
584	do {
585		tocopy = PAGE_SIZE - offset_in_page(from);
586		if (to > from)
587			tocopy = min(tocopy, to - from);
588		if (!tocopy)
589			break;
590
591		addr = rb->aux_pages[from >> PAGE_SHIFT];
592		addr += offset_in_page(from);
593
594		remainder = perf_output_copy(handle, addr, tocopy);
595		if (remainder)
596			return -EFAULT;
597
598		len += tocopy;
599		from += tocopy;
600		from &= (rb->aux_nr_pages << PAGE_SHIFT) - 1;
601	} while (to != from);
602
603	return len;
604}
605
606#define PERF_AUX_GFP	(GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
607
608static struct page *rb_alloc_aux_page(int node, int order)
609{
610	struct page *page;
611
612	if (order > MAX_ORDER)
613		order = MAX_ORDER;
614
615	do {
616		page = alloc_pages_node(node, PERF_AUX_GFP, order);
617	} while (!page && order--);
618
619	if (page && order) {
620		/*
621		 * Communicate the allocation size to the driver:
622		 * if we managed to secure a high-order allocation,
623		 * set its first page's private to this order;
624		 * !PagePrivate(page) means it's just a normal page.
625		 */
626		split_page(page, order);
627		SetPagePrivate(page);
628		set_page_private(page, order);
629	}
630
631	return page;
632}
633
634static void rb_free_aux_page(struct perf_buffer *rb, int idx)
635{
636	struct page *page = virt_to_page(rb->aux_pages[idx]);
637
638	ClearPagePrivate(page);
639	page->mapping = NULL;
640	__free_page(page);
641}
642
643static void __rb_free_aux(struct perf_buffer *rb)
644{
645	int pg;
646
647	/*
648	 * Should never happen, the last reference should be dropped from
649	 * perf_mmap_close() path, which first stops aux transactions (which
650	 * in turn are the atomic holders of aux_refcount) and then does the
651	 * last rb_free_aux().
652	 */
653	WARN_ON_ONCE(in_atomic());
654
655	if (rb->aux_priv) {
656		rb->free_aux(rb->aux_priv);
657		rb->free_aux = NULL;
658		rb->aux_priv = NULL;
659	}
660
661	if (rb->aux_nr_pages) {
662		for (pg = 0; pg < rb->aux_nr_pages; pg++)
663			rb_free_aux_page(rb, pg);
664
665		kfree(rb->aux_pages);
666		rb->aux_nr_pages = 0;
667	}
668}
669
670int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event,
671		 pgoff_t pgoff, int nr_pages, long watermark, int flags)
672{
673	bool overwrite = !(flags & RING_BUFFER_WRITABLE);
674	int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu);
675	int ret = -ENOMEM, max_order;
676
677	if (!has_aux(event))
678		return -EOPNOTSUPP;
679
680	if (!overwrite) {
681		/*
682		 * Watermark defaults to half the buffer, and so does the
683		 * max_order, to aid PMU drivers in double buffering.
684		 */
685		if (!watermark)
686			watermark = nr_pages << (PAGE_SHIFT - 1);
687
688		/*
689		 * Use aux_watermark as the basis for chunking to
690		 * help PMU drivers honor the watermark.
691		 */
692		max_order = get_order(watermark);
693	} else {
694		/*
695		 * We need to start with the max_order that fits in nr_pages,
696		 * not the other way around, hence ilog2() and not get_order.
697		 */
698		max_order = ilog2(nr_pages);
699		watermark = 0;
700	}
701
702	rb->aux_pages = kcalloc_node(nr_pages, sizeof(void *), GFP_KERNEL,
703				     node);
704	if (!rb->aux_pages)
705		return -ENOMEM;
706
707	rb->free_aux = event->pmu->free_aux;
708	for (rb->aux_nr_pages = 0; rb->aux_nr_pages < nr_pages;) {
709		struct page *page;
710		int last, order;
711
712		order = min(max_order, ilog2(nr_pages - rb->aux_nr_pages));
713		page = rb_alloc_aux_page(node, order);
714		if (!page)
715			goto out;
716
717		for (last = rb->aux_nr_pages + (1 << page_private(page));
718		     last > rb->aux_nr_pages; rb->aux_nr_pages++)
719			rb->aux_pages[rb->aux_nr_pages] = page_address(page++);
720	}
721
722	/*
723	 * In overwrite mode, PMUs that don't support SG may not handle more
724	 * than one contiguous allocation, since they rely on PMI to do double
725	 * buffering. In this case, the entire buffer has to be one contiguous
726	 * chunk.
727	 */
728	if ((event->pmu->capabilities & PERF_PMU_CAP_AUX_NO_SG) &&
729	    overwrite) {
730		struct page *page = virt_to_page(rb->aux_pages[0]);
731
732		if (page_private(page) != max_order)
733			goto out;
734	}
735
736	rb->aux_priv = event->pmu->setup_aux(event, rb->aux_pages, nr_pages,
737					     overwrite);
738	if (!rb->aux_priv)
739		goto out;
740
741	ret = 0;
742
743	/*
744	 * aux_pages (and pmu driver's private data, aux_priv) will be
745	 * referenced in both producer's and consumer's contexts, thus
746	 * we keep a refcount here to make sure either of the two can
747	 * reference them safely.
748	 */
749	refcount_set(&rb->aux_refcount, 1);
750
751	rb->aux_overwrite = overwrite;
752	rb->aux_watermark = watermark;
753
754out:
755	if (!ret)
756		rb->aux_pgoff = pgoff;
757	else
758		__rb_free_aux(rb);
759
760	return ret;
761}
762
763void rb_free_aux(struct perf_buffer *rb)
764{
765	if (refcount_dec_and_test(&rb->aux_refcount))
766		__rb_free_aux(rb);
767}
768
769#ifndef CONFIG_PERF_USE_VMALLOC
770
771/*
772 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
773 */
774
775static struct page *
776__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
777{
778	if (pgoff > rb->nr_pages)
779		return NULL;
780
781	if (pgoff == 0)
782		return virt_to_page(rb->user_page);
783
784	return virt_to_page(rb->data_pages[pgoff - 1]);
785}
786
787static void *perf_mmap_alloc_page(int cpu)
788{
789	struct page *page;
790	int node;
791
792	node = (cpu == -1) ? cpu : cpu_to_node(cpu);
793	page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
794	if (!page)
795		return NULL;
796
797	return page_address(page);
798}
799
800static void perf_mmap_free_page(void *addr)
801{
802	struct page *page = virt_to_page(addr);
803
804	page->mapping = NULL;
805	__free_page(page);
806}
807
808struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
809{
810	struct perf_buffer *rb;
811	unsigned long size;
812	int i, node;
813
814	size = sizeof(struct perf_buffer);
815	size += nr_pages * sizeof(void *);
816
817	if (order_base_2(size) >= PAGE_SHIFT+MAX_ORDER)
818		goto fail;
819
820	node = (cpu == -1) ? cpu : cpu_to_node(cpu);
821	rb = kzalloc_node(size, GFP_KERNEL, node);
822	if (!rb)
823		goto fail;
824
825	rb->user_page = perf_mmap_alloc_page(cpu);
826	if (!rb->user_page)
827		goto fail_user_page;
828
829	for (i = 0; i < nr_pages; i++) {
830		rb->data_pages[i] = perf_mmap_alloc_page(cpu);
831		if (!rb->data_pages[i])
832			goto fail_data_pages;
833	}
834
835	rb->nr_pages = nr_pages;
836
837	ring_buffer_init(rb, watermark, flags);
838
839	return rb;
840
841fail_data_pages:
842	for (i--; i >= 0; i--)
843		perf_mmap_free_page(rb->data_pages[i]);
844
845	perf_mmap_free_page(rb->user_page);
846
847fail_user_page:
848	kfree(rb);
849
850fail:
851	return NULL;
852}
853
854void rb_free(struct perf_buffer *rb)
 
 
 
 
 
 
 
 
855{
856	int i;
857
858	perf_mmap_free_page(rb->user_page);
859	for (i = 0; i < rb->nr_pages; i++)
860		perf_mmap_free_page(rb->data_pages[i]);
861	kfree(rb);
862}
863
864#else
865static struct page *
866__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
 
867{
868	/* The '>' counts in the user page. */
869	if (pgoff > data_page_nr(rb))
870		return NULL;
871
872	return vmalloc_to_page((void *)rb->user_page + pgoff * PAGE_SIZE);
873}
874
875static void perf_mmap_unmark_page(void *addr)
876{
877	struct page *page = vmalloc_to_page(addr);
878
879	page->mapping = NULL;
880}
881
882static void rb_free_work(struct work_struct *work)
883{
884	struct perf_buffer *rb;
885	void *base;
886	int i, nr;
887
888	rb = container_of(work, struct perf_buffer, work);
889	nr = data_page_nr(rb);
890
891	base = rb->user_page;
892	/* The '<=' counts in the user page. */
893	for (i = 0; i <= nr; i++)
894		perf_mmap_unmark_page(base + (i * PAGE_SIZE));
895
896	vfree(base);
897	kfree(rb);
898}
899
900void rb_free(struct perf_buffer *rb)
901{
902	schedule_work(&rb->work);
903}
904
905struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags)
906{
907	struct perf_buffer *rb;
908	unsigned long size;
909	void *all_buf;
910	int node;
911
912	size = sizeof(struct perf_buffer);
913	size += sizeof(void *);
914
915	node = (cpu == -1) ? cpu : cpu_to_node(cpu);
916	rb = kzalloc_node(size, GFP_KERNEL, node);
917	if (!rb)
918		goto fail;
919
920	INIT_WORK(&rb->work, rb_free_work);
921
922	all_buf = vmalloc_user((nr_pages + 1) * PAGE_SIZE);
923	if (!all_buf)
924		goto fail_all_buf;
925
926	rb->user_page = all_buf;
927	rb->data_pages[0] = all_buf + PAGE_SIZE;
928	if (nr_pages) {
929		rb->nr_pages = 1;
930		rb->page_order = ilog2(nr_pages);
931	}
932
933	ring_buffer_init(rb, watermark, flags);
934
935	return rb;
936
937fail_all_buf:
938	kfree(rb);
939
940fail:
941	return NULL;
942}
943
944#endif
945
946struct page *
947perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff)
948{
949	if (rb->aux_nr_pages) {
950		/* above AUX space */
951		if (pgoff > rb->aux_pgoff + rb->aux_nr_pages)
952			return NULL;
953
954		/* AUX space */
955		if (pgoff >= rb->aux_pgoff) {
956			int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages);
957			return virt_to_page(rb->aux_pages[aux_pgoff]);
958		}
959	}
960
961	return __perf_mmap_to_page(rb, pgoff);
962}