Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2#include <errno.h>
3#include <unistd.h>
4#include <sys/syscall.h>
5#include <perf/evsel.h>
6#include <perf/cpumap.h>
7#include <perf/threadmap.h>
8#include <linux/hash.h>
9#include <linux/list.h>
10#include <internal/evsel.h>
11#include <linux/zalloc.h>
12#include <stdlib.h>
13#include <internal/xyarray.h>
14#include <internal/cpumap.h>
15#include <internal/mmap.h>
16#include <internal/threadmap.h>
17#include <internal/lib.h>
18#include <linux/string.h>
19#include <sys/ioctl.h>
20#include <sys/mman.h>
21#include <asm/bug.h>
22
23void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr,
24 int idx)
25{
26 INIT_LIST_HEAD(&evsel->node);
27 INIT_LIST_HEAD(&evsel->per_stream_periods);
28 evsel->attr = *attr;
29 evsel->idx = idx;
30 evsel->leader = evsel;
31}
32
33struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr)
34{
35 struct perf_evsel *evsel = zalloc(sizeof(*evsel));
36
37 if (evsel != NULL)
38 perf_evsel__init(evsel, attr, 0);
39
40 return evsel;
41}
42
43void perf_evsel__delete(struct perf_evsel *evsel)
44{
45 free(evsel);
46}
47
48#define FD(_evsel, _cpu_map_idx, _thread) \
49 ((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread))
50#define MMAP(_evsel, _cpu_map_idx, _thread) \
51 (_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \
52 : NULL)
53
54int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
55{
56 evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
57
58 if (evsel->fd) {
59 int idx, thread;
60
61 for (idx = 0; idx < ncpus; idx++) {
62 for (thread = 0; thread < nthreads; thread++) {
63 int *fd = FD(evsel, idx, thread);
64
65 if (fd)
66 *fd = -1;
67 }
68 }
69 }
70
71 return evsel->fd != NULL ? 0 : -ENOMEM;
72}
73
74static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads)
75{
76 evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap));
77
78 return evsel->mmap != NULL ? 0 : -ENOMEM;
79}
80
81static int
82sys_perf_event_open(struct perf_event_attr *attr,
83 pid_t pid, struct perf_cpu cpu, int group_fd,
84 unsigned long flags)
85{
86 return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags);
87}
88
89static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd)
90{
91 struct perf_evsel *leader = evsel->leader;
92 int *fd;
93
94 if (evsel == leader) {
95 *group_fd = -1;
96 return 0;
97 }
98
99 /*
100 * Leader must be already processed/open,
101 * if not it's a bug.
102 */
103 if (!leader->fd)
104 return -ENOTCONN;
105
106 fd = FD(leader, cpu_map_idx, thread);
107 if (fd == NULL || *fd == -1)
108 return -EBADF;
109
110 *group_fd = *fd;
111
112 return 0;
113}
114
115int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus,
116 struct perf_thread_map *threads)
117{
118 struct perf_cpu cpu;
119 int idx, thread, err = 0;
120
121 if (cpus == NULL) {
122 static struct perf_cpu_map *empty_cpu_map;
123
124 if (empty_cpu_map == NULL) {
125 empty_cpu_map = perf_cpu_map__new_any_cpu();
126 if (empty_cpu_map == NULL)
127 return -ENOMEM;
128 }
129
130 cpus = empty_cpu_map;
131 }
132
133 if (threads == NULL) {
134 static struct perf_thread_map *empty_thread_map;
135
136 if (empty_thread_map == NULL) {
137 empty_thread_map = perf_thread_map__new_dummy();
138 if (empty_thread_map == NULL)
139 return -ENOMEM;
140 }
141
142 threads = empty_thread_map;
143 }
144
145 if (evsel->fd == NULL &&
146 perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0)
147 return -ENOMEM;
148
149 perf_cpu_map__for_each_cpu(cpu, idx, cpus) {
150 for (thread = 0; thread < threads->nr; thread++) {
151 int fd, group_fd, *evsel_fd;
152
153 evsel_fd = FD(evsel, idx, thread);
154 if (evsel_fd == NULL) {
155 err = -EINVAL;
156 goto out;
157 }
158
159 err = get_group_fd(evsel, idx, thread, &group_fd);
160 if (err < 0)
161 goto out;
162
163 fd = sys_perf_event_open(&evsel->attr,
164 threads->map[thread].pid,
165 cpu, group_fd, 0);
166
167 if (fd < 0) {
168 err = -errno;
169 goto out;
170 }
171
172 *evsel_fd = fd;
173 }
174 }
175out:
176 if (err)
177 perf_evsel__close(evsel);
178
179 return err;
180}
181
182static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx)
183{
184 int thread;
185
186 for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) {
187 int *fd = FD(evsel, cpu_map_idx, thread);
188
189 if (fd && *fd >= 0) {
190 close(*fd);
191 *fd = -1;
192 }
193 }
194}
195
196void perf_evsel__close_fd(struct perf_evsel *evsel)
197{
198 for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++)
199 perf_evsel__close_fd_cpu(evsel, idx);
200}
201
202void perf_evsel__free_fd(struct perf_evsel *evsel)
203{
204 xyarray__delete(evsel->fd);
205 evsel->fd = NULL;
206}
207
208void perf_evsel__close(struct perf_evsel *evsel)
209{
210 if (evsel->fd == NULL)
211 return;
212
213 perf_evsel__close_fd(evsel);
214 perf_evsel__free_fd(evsel);
215}
216
217void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx)
218{
219 if (evsel->fd == NULL)
220 return;
221
222 perf_evsel__close_fd_cpu(evsel, cpu_map_idx);
223}
224
225void perf_evsel__munmap(struct perf_evsel *evsel)
226{
227 int idx, thread;
228
229 if (evsel->fd == NULL || evsel->mmap == NULL)
230 return;
231
232 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
233 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
234 int *fd = FD(evsel, idx, thread);
235
236 if (fd == NULL || *fd < 0)
237 continue;
238
239 perf_mmap__munmap(MMAP(evsel, idx, thread));
240 }
241 }
242
243 xyarray__delete(evsel->mmap);
244 evsel->mmap = NULL;
245}
246
247int perf_evsel__mmap(struct perf_evsel *evsel, int pages)
248{
249 int ret, idx, thread;
250 struct perf_mmap_param mp = {
251 .prot = PROT_READ | PROT_WRITE,
252 .mask = (pages * page_size) - 1,
253 };
254
255 if (evsel->fd == NULL || evsel->mmap)
256 return -EINVAL;
257
258 if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0)
259 return -ENOMEM;
260
261 for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
262 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
263 int *fd = FD(evsel, idx, thread);
264 struct perf_mmap *map;
265 struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx);
266
267 if (fd == NULL || *fd < 0)
268 continue;
269
270 map = MMAP(evsel, idx, thread);
271 perf_mmap__init(map, NULL, false, NULL);
272
273 ret = perf_mmap__mmap(map, &mp, *fd, cpu);
274 if (ret) {
275 perf_evsel__munmap(evsel);
276 return ret;
277 }
278 }
279 }
280
281 return 0;
282}
283
284void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread)
285{
286 int *fd = FD(evsel, cpu_map_idx, thread);
287
288 if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL)
289 return NULL;
290
291 return MMAP(evsel, cpu_map_idx, thread)->base;
292}
293
294int perf_evsel__read_size(struct perf_evsel *evsel)
295{
296 u64 read_format = evsel->attr.read_format;
297 int entry = sizeof(u64); /* value */
298 int size = 0;
299 int nr = 1;
300
301 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
302 size += sizeof(u64);
303
304 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
305 size += sizeof(u64);
306
307 if (read_format & PERF_FORMAT_ID)
308 entry += sizeof(u64);
309
310 if (read_format & PERF_FORMAT_LOST)
311 entry += sizeof(u64);
312
313 if (read_format & PERF_FORMAT_GROUP) {
314 nr = evsel->nr_members;
315 size += sizeof(u64);
316 }
317
318 size += entry * nr;
319 return size;
320}
321
322/* This only reads values for the leader */
323static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
324 int thread, struct perf_counts_values *count)
325{
326 size_t size = perf_evsel__read_size(evsel);
327 int *fd = FD(evsel, cpu_map_idx, thread);
328 u64 read_format = evsel->attr.read_format;
329 u64 *data;
330 int idx = 1;
331
332 if (fd == NULL || *fd < 0)
333 return -EINVAL;
334
335 data = calloc(1, size);
336 if (data == NULL)
337 return -ENOMEM;
338
339 if (readn(*fd, data, size) <= 0) {
340 free(data);
341 return -errno;
342 }
343
344 /*
345 * This reads only the leader event intentionally since we don't have
346 * perf counts values for sibling events.
347 */
348 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
349 count->ena = data[idx++];
350 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
351 count->run = data[idx++];
352
353 /* value is always available */
354 count->val = data[idx++];
355 if (read_format & PERF_FORMAT_ID)
356 count->id = data[idx++];
357 if (read_format & PERF_FORMAT_LOST)
358 count->lost = data[idx++];
359
360 free(data);
361 return 0;
362}
363
364/*
365 * The perf read format is very flexible. It needs to set the proper
366 * values according to the read format.
367 */
368static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf,
369 struct perf_counts_values *count)
370{
371 u64 read_format = evsel->attr.read_format;
372 int n = 0;
373
374 count->val = buf[n++];
375
376 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
377 count->ena = buf[n++];
378
379 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
380 count->run = buf[n++];
381
382 if (read_format & PERF_FORMAT_ID)
383 count->id = buf[n++];
384
385 if (read_format & PERF_FORMAT_LOST)
386 count->lost = buf[n++];
387}
388
389int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread,
390 struct perf_counts_values *count)
391{
392 size_t size = perf_evsel__read_size(evsel);
393 int *fd = FD(evsel, cpu_map_idx, thread);
394 u64 read_format = evsel->attr.read_format;
395 struct perf_counts_values buf;
396
397 memset(count, 0, sizeof(*count));
398
399 if (fd == NULL || *fd < 0)
400 return -EINVAL;
401
402 if (read_format & PERF_FORMAT_GROUP)
403 return perf_evsel__read_group(evsel, cpu_map_idx, thread, count);
404
405 if (MMAP(evsel, cpu_map_idx, thread) &&
406 !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) &&
407 !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count))
408 return 0;
409
410 if (readn(*fd, buf.values, size) <= 0)
411 return -errno;
412
413 perf_evsel__adjust_values(evsel, buf.values, count);
414 return 0;
415}
416
417static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg,
418 int cpu_map_idx, int thread)
419{
420 int *fd = FD(evsel, cpu_map_idx, thread);
421
422 if (fd == NULL || *fd < 0)
423 return -1;
424
425 return ioctl(*fd, ioc, arg);
426}
427
428static int perf_evsel__run_ioctl(struct perf_evsel *evsel,
429 int ioc, void *arg,
430 int cpu_map_idx)
431{
432 int thread;
433
434 for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
435 int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread);
436
437 if (err)
438 return err;
439 }
440
441 return 0;
442}
443
444int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
445{
446 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx);
447}
448
449int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread)
450{
451 struct perf_cpu cpu __maybe_unused;
452 int idx;
453 int err;
454
455 perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) {
456 err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread);
457 if (err)
458 return err;
459 }
460
461 return 0;
462}
463
464int perf_evsel__enable(struct perf_evsel *evsel)
465{
466 int i;
467 int err = 0;
468
469 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
470 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i);
471 return err;
472}
473
474int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
475{
476 return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx);
477}
478
479int perf_evsel__disable(struct perf_evsel *evsel)
480{
481 int i;
482 int err = 0;
483
484 for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
485 err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i);
486 return err;
487}
488
489int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter)
490{
491 int err = 0, i;
492
493 for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++)
494 err = perf_evsel__run_ioctl(evsel,
495 PERF_EVENT_IOC_SET_FILTER,
496 (void *)filter, i);
497 return err;
498}
499
500struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel)
501{
502 return evsel->cpus;
503}
504
505struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel)
506{
507 return evsel->threads;
508}
509
510struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel)
511{
512 return &evsel->attr;
513}
514
515int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
516{
517 if (ncpus == 0 || nthreads == 0)
518 return 0;
519
520 evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
521 if (evsel->sample_id == NULL)
522 return -ENOMEM;
523
524 evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
525 if (evsel->id == NULL) {
526 xyarray__delete(evsel->sample_id);
527 evsel->sample_id = NULL;
528 return -ENOMEM;
529 }
530
531 return 0;
532}
533
534void perf_evsel__free_id(struct perf_evsel *evsel)
535{
536 struct perf_sample_id_period *pos, *n;
537
538 xyarray__delete(evsel->sample_id);
539 evsel->sample_id = NULL;
540 zfree(&evsel->id);
541 evsel->ids = 0;
542
543 perf_evsel_for_each_per_thread_period_safe(evsel, n, pos) {
544 list_del_init(&pos->node);
545 free(pos);
546 }
547}
548
549bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel)
550{
551 return (evsel->attr.sample_type & PERF_SAMPLE_READ) &&
552 (evsel->attr.sample_type & PERF_SAMPLE_TID) &&
553 evsel->attr.inherit;
554}
555
556u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, bool per_thread)
557{
558 struct hlist_head *head;
559 struct perf_sample_id_period *res;
560 int hash;
561
562 if (!per_thread)
563 return &sid->period;
564
565 hash = hash_32(tid, PERF_SAMPLE_ID__HLIST_BITS);
566 head = &sid->periods[hash];
567
568 hlist_for_each_entry(res, head, hnode)
569 if (res->tid == tid)
570 return &res->period;
571
572 if (sid->evsel == NULL)
573 return NULL;
574
575 res = zalloc(sizeof(struct perf_sample_id_period));
576 if (res == NULL)
577 return NULL;
578
579 INIT_LIST_HEAD(&res->node);
580 res->tid = tid;
581
582 list_add_tail(&res->node, &sid->evsel->per_stream_periods);
583 hlist_add_head(&res->hnode, &sid->periods[hash]);
584
585 return &res->period;
586}
587
588void perf_counts_values__scale(struct perf_counts_values *count,
589 bool scale, __s8 *pscaled)
590{
591 s8 scaled = 0;
592
593 if (scale) {
594 if (count->run == 0) {
595 scaled = -1;
596 count->val = 0;
597 } else if (count->run < count->ena) {
598 scaled = 1;
599 count->val = (u64)((double)count->val * count->ena / count->run);
600 }
601 }
602
603 if (pscaled)
604 *pscaled = scaled;
605}