Loading...
Note: File does not exist in v5.4.
1/*
2 * Copyright IBM Corp. 2010
3 * Author: Heinz Graalfs <graalfs@de.ibm.com>
4 */
5
6#include <linux/kernel_stat.h>
7#include <linux/kernel.h>
8#include <linux/module.h>
9#include <linux/smp.h>
10#include <linux/errno.h>
11#include <linux/workqueue.h>
12#include <linux/interrupt.h>
13#include <linux/notifier.h>
14#include <linux/cpu.h>
15#include <linux/semaphore.h>
16#include <linux/oom.h>
17#include <linux/oprofile.h>
18
19#include <asm/facility.h>
20#include <asm/cpu_mf.h>
21#include <asm/irq.h>
22
23#include "hwsampler.h"
24#include "op_counter.h"
25
26#define MAX_NUM_SDB 511
27#define MIN_NUM_SDB 1
28
29DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
30
31struct hws_execute_parms {
32 void *buffer;
33 signed int rc;
34};
35
36DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
37EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
38
39static DEFINE_MUTEX(hws_sem);
40static DEFINE_MUTEX(hws_sem_oom);
41
42static unsigned char hws_flush_all;
43static unsigned int hws_oom;
44static unsigned int hws_alert;
45static struct workqueue_struct *hws_wq;
46
47static unsigned int hws_state;
48enum {
49 HWS_INIT = 1,
50 HWS_DEALLOCATED,
51 HWS_STOPPED,
52 HWS_STARTED,
53 HWS_STOPPING };
54
55/* set to 1 if called by kernel during memory allocation */
56static unsigned char oom_killer_was_active;
57/* size of SDBT and SDB as of allocate API */
58static unsigned long num_sdbt = 100;
59static unsigned long num_sdb = 511;
60/* sampling interval (machine cycles) */
61static unsigned long interval;
62
63static unsigned long min_sampler_rate;
64static unsigned long max_sampler_rate;
65
66static void execute_qsi(void *parms)
67{
68 struct hws_execute_parms *ep = parms;
69
70 ep->rc = qsi(ep->buffer);
71}
72
73static void execute_ssctl(void *parms)
74{
75 struct hws_execute_parms *ep = parms;
76
77 ep->rc = lsctl(ep->buffer);
78}
79
80static int smp_ctl_ssctl_stop(int cpu)
81{
82 int rc;
83 struct hws_execute_parms ep;
84 struct hws_cpu_buffer *cb;
85
86 cb = &per_cpu(sampler_cpu_buffer, cpu);
87
88 cb->ssctl.es = 0;
89 cb->ssctl.cs = 0;
90
91 ep.buffer = &cb->ssctl;
92 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
93 rc = ep.rc;
94 if (rc) {
95 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
96 dump_stack();
97 }
98
99 ep.buffer = &cb->qsi;
100 smp_call_function_single(cpu, execute_qsi, &ep, 1);
101
102 if (cb->qsi.es || cb->qsi.cs) {
103 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
104 dump_stack();
105 }
106
107 return rc;
108}
109
110static int smp_ctl_ssctl_deactivate(int cpu)
111{
112 int rc;
113 struct hws_execute_parms ep;
114 struct hws_cpu_buffer *cb;
115
116 cb = &per_cpu(sampler_cpu_buffer, cpu);
117
118 cb->ssctl.es = 1;
119 cb->ssctl.cs = 0;
120
121 ep.buffer = &cb->ssctl;
122 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
123 rc = ep.rc;
124 if (rc)
125 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
126
127 ep.buffer = &cb->qsi;
128 smp_call_function_single(cpu, execute_qsi, &ep, 1);
129
130 if (cb->qsi.cs)
131 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
132
133 return rc;
134}
135
136static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
137{
138 int rc;
139 struct hws_execute_parms ep;
140 struct hws_cpu_buffer *cb;
141
142 cb = &per_cpu(sampler_cpu_buffer, cpu);
143
144 cb->ssctl.h = 1;
145 cb->ssctl.tear = cb->first_sdbt;
146 cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
147 cb->ssctl.interval = interval;
148 cb->ssctl.es = 1;
149 cb->ssctl.cs = 1;
150
151 ep.buffer = &cb->ssctl;
152 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
153 rc = ep.rc;
154 if (rc)
155 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
156
157 ep.buffer = &cb->qsi;
158 smp_call_function_single(cpu, execute_qsi, &ep, 1);
159 if (ep.rc)
160 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
161
162 return rc;
163}
164
165static int smp_ctl_qsi(int cpu)
166{
167 struct hws_execute_parms ep;
168 struct hws_cpu_buffer *cb;
169
170 cb = &per_cpu(sampler_cpu_buffer, cpu);
171
172 ep.buffer = &cb->qsi;
173 smp_call_function_single(cpu, execute_qsi, &ep, 1);
174
175 return ep.rc;
176}
177
178static void hws_ext_handler(struct ext_code ext_code,
179 unsigned int param32, unsigned long param64)
180{
181 struct hws_cpu_buffer *cb = &__get_cpu_var(sampler_cpu_buffer);
182
183 if (!(param32 & CPU_MF_INT_SF_MASK))
184 return;
185
186 if (!hws_alert)
187 return;
188
189 inc_irq_stat(IRQEXT_CMS);
190 atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
191
192 if (hws_wq)
193 queue_work(hws_wq, &cb->worker);
194}
195
196static void worker(struct work_struct *work);
197
198static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
199 unsigned long *dear);
200
201static void init_all_cpu_buffers(void)
202{
203 int cpu;
204 struct hws_cpu_buffer *cb;
205
206 for_each_online_cpu(cpu) {
207 cb = &per_cpu(sampler_cpu_buffer, cpu);
208 memset(cb, 0, sizeof(struct hws_cpu_buffer));
209 }
210}
211
212static int prepare_cpu_buffers(void)
213{
214 int cpu;
215 int rc;
216 struct hws_cpu_buffer *cb;
217
218 rc = 0;
219 for_each_online_cpu(cpu) {
220 cb = &per_cpu(sampler_cpu_buffer, cpu);
221 atomic_set(&cb->ext_params, 0);
222 cb->worker_entry = 0;
223 cb->sample_overflow = 0;
224 cb->req_alert = 0;
225 cb->incorrect_sdbt_entry = 0;
226 cb->invalid_entry_address = 0;
227 cb->loss_of_sample_data = 0;
228 cb->sample_auth_change_alert = 0;
229 cb->finish = 0;
230 cb->oom = 0;
231 cb->stop_mode = 0;
232 }
233
234 return rc;
235}
236
237/*
238 * allocate_sdbt() - allocate sampler memory
239 * @cpu: the cpu for which sampler memory is allocated
240 *
241 * A 4K page is allocated for each requested SDBT.
242 * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
243 * Set ALERT_REQ mask in each SDBs trailer.
244 * Returns zero if successful, <0 otherwise.
245 */
246static int allocate_sdbt(int cpu)
247{
248 int j, k, rc;
249 unsigned long *sdbt;
250 unsigned long sdb;
251 unsigned long *tail;
252 unsigned long *trailer;
253 struct hws_cpu_buffer *cb;
254
255 cb = &per_cpu(sampler_cpu_buffer, cpu);
256
257 if (cb->first_sdbt)
258 return -EINVAL;
259
260 sdbt = NULL;
261 tail = sdbt;
262
263 for (j = 0; j < num_sdbt; j++) {
264 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
265
266 mutex_lock(&hws_sem_oom);
267 /* OOM killer might have been activated */
268 barrier();
269 if (oom_killer_was_active || !sdbt) {
270 if (sdbt)
271 free_page((unsigned long)sdbt);
272
273 goto allocate_sdbt_error;
274 }
275 if (cb->first_sdbt == 0)
276 cb->first_sdbt = (unsigned long)sdbt;
277
278 /* link current page to tail of chain */
279 if (tail)
280 *tail = (unsigned long)(void *)sdbt + 1;
281
282 mutex_unlock(&hws_sem_oom);
283
284 for (k = 0; k < num_sdb; k++) {
285 /* get and set SDB page */
286 sdb = get_zeroed_page(GFP_KERNEL);
287
288 mutex_lock(&hws_sem_oom);
289 /* OOM killer might have been activated */
290 barrier();
291 if (oom_killer_was_active || !sdb) {
292 if (sdb)
293 free_page(sdb);
294
295 goto allocate_sdbt_error;
296 }
297 *sdbt = sdb;
298 trailer = trailer_entry_ptr(*sdbt);
299 *trailer = SDB_TE_ALERT_REQ_MASK;
300 sdbt++;
301 mutex_unlock(&hws_sem_oom);
302 }
303 tail = sdbt;
304 }
305 mutex_lock(&hws_sem_oom);
306 if (oom_killer_was_active)
307 goto allocate_sdbt_error;
308
309 rc = 0;
310 if (tail)
311 *tail = (unsigned long)
312 ((void *)cb->first_sdbt) + 1;
313
314allocate_sdbt_exit:
315 mutex_unlock(&hws_sem_oom);
316 return rc;
317
318allocate_sdbt_error:
319 rc = -ENOMEM;
320 goto allocate_sdbt_exit;
321}
322
323/*
324 * deallocate_sdbt() - deallocate all sampler memory
325 *
326 * For each online CPU all SDBT trees are deallocated.
327 * Returns the number of freed pages.
328 */
329static int deallocate_sdbt(void)
330{
331 int cpu;
332 int counter;
333
334 counter = 0;
335
336 for_each_online_cpu(cpu) {
337 unsigned long start;
338 unsigned long sdbt;
339 unsigned long *curr;
340 struct hws_cpu_buffer *cb;
341
342 cb = &per_cpu(sampler_cpu_buffer, cpu);
343
344 if (!cb->first_sdbt)
345 continue;
346
347 sdbt = cb->first_sdbt;
348 curr = (unsigned long *) sdbt;
349 start = sdbt;
350
351 /* we'll free the SDBT after all SDBs are processed... */
352 while (1) {
353 if (!*curr || !sdbt)
354 break;
355
356 /* watch for link entry reset if found */
357 if (is_link_entry(curr)) {
358 curr = get_next_sdbt(curr);
359 if (sdbt)
360 free_page(sdbt);
361
362 /* we are done if we reach the start */
363 if ((unsigned long) curr == start)
364 break;
365 else
366 sdbt = (unsigned long) curr;
367 } else {
368 /* process SDB pointer */
369 if (*curr) {
370 free_page(*curr);
371 curr++;
372 }
373 }
374 counter++;
375 }
376 cb->first_sdbt = 0;
377 }
378 return counter;
379}
380
381static int start_sampling(int cpu)
382{
383 int rc;
384 struct hws_cpu_buffer *cb;
385
386 cb = &per_cpu(sampler_cpu_buffer, cpu);
387 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
388 if (rc) {
389 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
390 goto start_exit;
391 }
392
393 rc = -EINVAL;
394 if (!cb->qsi.es) {
395 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
396 goto start_exit;
397 }
398
399 if (!cb->qsi.cs) {
400 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
401 goto start_exit;
402 }
403
404 printk(KERN_INFO
405 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
406 cpu, interval);
407
408 rc = 0;
409
410start_exit:
411 return rc;
412}
413
414static int stop_sampling(int cpu)
415{
416 unsigned long v;
417 int rc;
418 struct hws_cpu_buffer *cb;
419
420 rc = smp_ctl_qsi(cpu);
421 WARN_ON(rc);
422
423 cb = &per_cpu(sampler_cpu_buffer, cpu);
424 if (!rc && !cb->qsi.es)
425 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
426
427 rc = smp_ctl_ssctl_stop(cpu);
428 if (rc) {
429 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
430 cpu, rc);
431 goto stop_exit;
432 }
433
434 printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
435
436stop_exit:
437 v = cb->req_alert;
438 if (v)
439 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
440 " count=%lu.\n", cpu, v);
441
442 v = cb->loss_of_sample_data;
443 if (v)
444 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
445 " count=%lu.\n", cpu, v);
446
447 v = cb->invalid_entry_address;
448 if (v)
449 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
450 " count=%lu.\n", cpu, v);
451
452 v = cb->incorrect_sdbt_entry;
453 if (v)
454 printk(KERN_ERR
455 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
456 " count=%lu.\n", cpu, v);
457
458 v = cb->sample_auth_change_alert;
459 if (v)
460 printk(KERN_ERR
461 "hwsampler: CPU %d CPUMF Sample authorization change,"
462 " count=%lu.\n", cpu, v);
463
464 return rc;
465}
466
467static int check_hardware_prerequisites(void)
468{
469 if (!test_facility(68))
470 return -EOPNOTSUPP;
471 return 0;
472}
473/*
474 * hws_oom_callback() - the OOM callback function
475 *
476 * In case the callback is invoked during memory allocation for the
477 * hw sampler, all obtained memory is deallocated and a flag is set
478 * so main sampler memory allocation can exit with a failure code.
479 * In case the callback is invoked during sampling the hw sampler
480 * is deactivated for all CPUs.
481 */
482static int hws_oom_callback(struct notifier_block *nfb,
483 unsigned long dummy, void *parm)
484{
485 unsigned long *freed;
486 int cpu;
487 struct hws_cpu_buffer *cb;
488
489 freed = parm;
490
491 mutex_lock(&hws_sem_oom);
492
493 if (hws_state == HWS_DEALLOCATED) {
494 /* during memory allocation */
495 if (oom_killer_was_active == 0) {
496 oom_killer_was_active = 1;
497 *freed += deallocate_sdbt();
498 }
499 } else {
500 int i;
501 cpu = get_cpu();
502 cb = &per_cpu(sampler_cpu_buffer, cpu);
503
504 if (!cb->oom) {
505 for_each_online_cpu(i) {
506 smp_ctl_ssctl_deactivate(i);
507 cb->oom = 1;
508 }
509 cb->finish = 1;
510
511 printk(KERN_INFO
512 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
513 cpu);
514 }
515 }
516
517 mutex_unlock(&hws_sem_oom);
518
519 return NOTIFY_OK;
520}
521
522static struct notifier_block hws_oom_notifier = {
523 .notifier_call = hws_oom_callback
524};
525
526static int hws_cpu_callback(struct notifier_block *nfb,
527 unsigned long action, void *hcpu)
528{
529 /* We do not have sampler space available for all possible CPUs.
530 All CPUs should be online when hw sampling is activated. */
531 return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
532}
533
534static struct notifier_block hws_cpu_notifier = {
535 .notifier_call = hws_cpu_callback
536};
537
538/**
539 * hwsampler_deactivate() - set hardware sampling temporarily inactive
540 * @cpu: specifies the CPU to be set inactive.
541 *
542 * Returns 0 on success, !0 on failure.
543 */
544int hwsampler_deactivate(unsigned int cpu)
545{
546 /*
547 * Deactivate hw sampling temporarily and flush the buffer
548 * by pushing all the pending samples to oprofile buffer.
549 *
550 * This function can be called under one of the following conditions:
551 * Memory unmap, task is exiting.
552 */
553 int rc;
554 struct hws_cpu_buffer *cb;
555
556 rc = 0;
557 mutex_lock(&hws_sem);
558
559 cb = &per_cpu(sampler_cpu_buffer, cpu);
560 if (hws_state == HWS_STARTED) {
561 rc = smp_ctl_qsi(cpu);
562 WARN_ON(rc);
563 if (cb->qsi.cs) {
564 rc = smp_ctl_ssctl_deactivate(cpu);
565 if (rc) {
566 printk(KERN_INFO
567 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
568 cb->finish = 1;
569 hws_state = HWS_STOPPING;
570 } else {
571 hws_flush_all = 1;
572 /* Add work to queue to read pending samples.*/
573 queue_work_on(cpu, hws_wq, &cb->worker);
574 }
575 }
576 }
577 mutex_unlock(&hws_sem);
578
579 if (hws_wq)
580 flush_workqueue(hws_wq);
581
582 return rc;
583}
584
585/**
586 * hwsampler_activate() - activate/resume hardware sampling which was deactivated
587 * @cpu: specifies the CPU to be set active.
588 *
589 * Returns 0 on success, !0 on failure.
590 */
591int hwsampler_activate(unsigned int cpu)
592{
593 /*
594 * Re-activate hw sampling. This should be called in pair with
595 * hwsampler_deactivate().
596 */
597 int rc;
598 struct hws_cpu_buffer *cb;
599
600 rc = 0;
601 mutex_lock(&hws_sem);
602
603 cb = &per_cpu(sampler_cpu_buffer, cpu);
604 if (hws_state == HWS_STARTED) {
605 rc = smp_ctl_qsi(cpu);
606 WARN_ON(rc);
607 if (!cb->qsi.cs) {
608 hws_flush_all = 0;
609 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
610 if (rc) {
611 printk(KERN_ERR
612 "CPU %d, CPUMF activate sampling failed.\n",
613 cpu);
614 }
615 }
616 }
617
618 mutex_unlock(&hws_sem);
619
620 return rc;
621}
622
623static int check_qsi_on_setup(void)
624{
625 int rc;
626 unsigned int cpu;
627 struct hws_cpu_buffer *cb;
628
629 for_each_online_cpu(cpu) {
630 cb = &per_cpu(sampler_cpu_buffer, cpu);
631 rc = smp_ctl_qsi(cpu);
632 WARN_ON(rc);
633 if (rc)
634 return -EOPNOTSUPP;
635
636 if (!cb->qsi.as) {
637 printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
638 return -EINVAL;
639 }
640
641 if (cb->qsi.es) {
642 printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
643 rc = smp_ctl_ssctl_stop(cpu);
644 if (rc)
645 return -EINVAL;
646
647 printk(KERN_INFO
648 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
649 }
650 }
651 return 0;
652}
653
654static int check_qsi_on_start(void)
655{
656 unsigned int cpu;
657 int rc;
658 struct hws_cpu_buffer *cb;
659
660 for_each_online_cpu(cpu) {
661 cb = &per_cpu(sampler_cpu_buffer, cpu);
662 rc = smp_ctl_qsi(cpu);
663 WARN_ON(rc);
664
665 if (!cb->qsi.as)
666 return -EINVAL;
667
668 if (cb->qsi.es)
669 return -EINVAL;
670
671 if (cb->qsi.cs)
672 return -EINVAL;
673 }
674 return 0;
675}
676
677static void worker_on_start(unsigned int cpu)
678{
679 struct hws_cpu_buffer *cb;
680
681 cb = &per_cpu(sampler_cpu_buffer, cpu);
682 cb->worker_entry = cb->first_sdbt;
683}
684
685static int worker_check_error(unsigned int cpu, int ext_params)
686{
687 int rc;
688 unsigned long *sdbt;
689 struct hws_cpu_buffer *cb;
690
691 rc = 0;
692 cb = &per_cpu(sampler_cpu_buffer, cpu);
693 sdbt = (unsigned long *) cb->worker_entry;
694
695 if (!sdbt || !*sdbt)
696 return -EINVAL;
697
698 if (ext_params & CPU_MF_INT_SF_PRA)
699 cb->req_alert++;
700
701 if (ext_params & CPU_MF_INT_SF_LSDA)
702 cb->loss_of_sample_data++;
703
704 if (ext_params & CPU_MF_INT_SF_IAE) {
705 cb->invalid_entry_address++;
706 rc = -EINVAL;
707 }
708
709 if (ext_params & CPU_MF_INT_SF_ISE) {
710 cb->incorrect_sdbt_entry++;
711 rc = -EINVAL;
712 }
713
714 if (ext_params & CPU_MF_INT_SF_SACA) {
715 cb->sample_auth_change_alert++;
716 rc = -EINVAL;
717 }
718
719 return rc;
720}
721
722static void worker_on_finish(unsigned int cpu)
723{
724 int rc, i;
725 struct hws_cpu_buffer *cb;
726
727 cb = &per_cpu(sampler_cpu_buffer, cpu);
728
729 if (cb->finish) {
730 rc = smp_ctl_qsi(cpu);
731 WARN_ON(rc);
732 if (cb->qsi.es) {
733 printk(KERN_INFO
734 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
735 cpu);
736 rc = smp_ctl_ssctl_stop(cpu);
737 if (rc)
738 printk(KERN_INFO
739 "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
740 cpu);
741
742 for_each_online_cpu(i) {
743 if (i == cpu)
744 continue;
745 if (!cb->finish) {
746 cb->finish = 1;
747 queue_work_on(i, hws_wq,
748 &cb->worker);
749 }
750 }
751 }
752 }
753}
754
755static void worker_on_interrupt(unsigned int cpu)
756{
757 unsigned long *sdbt;
758 unsigned char done;
759 struct hws_cpu_buffer *cb;
760
761 cb = &per_cpu(sampler_cpu_buffer, cpu);
762
763 sdbt = (unsigned long *) cb->worker_entry;
764
765 done = 0;
766 /* do not proceed if stop was entered,
767 * forget the buffers not yet processed */
768 while (!done && !cb->stop_mode) {
769 unsigned long *trailer;
770 struct hws_trailer_entry *te;
771 unsigned long *dear = 0;
772
773 trailer = trailer_entry_ptr(*sdbt);
774 /* leave loop if no more work to do */
775 if (!(*trailer & SDB_TE_BUFFER_FULL_MASK)) {
776 done = 1;
777 if (!hws_flush_all)
778 continue;
779 }
780
781 te = (struct hws_trailer_entry *)trailer;
782 cb->sample_overflow += te->overflow;
783
784 add_samples_to_oprofile(cpu, sdbt, dear);
785
786 /* reset trailer */
787 xchg((unsigned char *) te, 0x40);
788
789 /* advance to next sdb slot in current sdbt */
790 sdbt++;
791 /* in case link bit is set use address w/o link bit */
792 if (is_link_entry(sdbt))
793 sdbt = get_next_sdbt(sdbt);
794
795 cb->worker_entry = (unsigned long)sdbt;
796 }
797}
798
799static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
800 unsigned long *dear)
801{
802 struct hws_basic_entry *sample_data_ptr;
803 unsigned long *trailer;
804
805 trailer = trailer_entry_ptr(*sdbt);
806 if (dear) {
807 if (dear > trailer)
808 return;
809 trailer = dear;
810 }
811
812 sample_data_ptr = (struct hws_basic_entry *)(*sdbt);
813
814 while ((unsigned long *)sample_data_ptr < trailer) {
815 struct pt_regs *regs = NULL;
816 struct task_struct *tsk = NULL;
817
818 /*
819 * Check sampling mode, 1 indicates basic (=customer) sampling
820 * mode.
821 */
822 if (sample_data_ptr->def != 1) {
823 /* sample slot is not yet written */
824 break;
825 } else {
826 /* make sure we don't use it twice,
827 * the next time the sampler will set it again */
828 sample_data_ptr->def = 0;
829 }
830
831 /* Get pt_regs. */
832 if (sample_data_ptr->P == 1) {
833 /* userspace sample */
834 unsigned int pid = sample_data_ptr->prim_asn;
835 if (!counter_config.user)
836 goto skip_sample;
837 rcu_read_lock();
838 tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
839 if (tsk)
840 regs = task_pt_regs(tsk);
841 rcu_read_unlock();
842 } else {
843 /* kernelspace sample */
844 if (!counter_config.kernel)
845 goto skip_sample;
846 regs = task_pt_regs(current);
847 }
848
849 mutex_lock(&hws_sem);
850 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
851 !sample_data_ptr->P, tsk);
852 mutex_unlock(&hws_sem);
853 skip_sample:
854 sample_data_ptr++;
855 }
856}
857
858static void worker(struct work_struct *work)
859{
860 unsigned int cpu;
861 int ext_params;
862 struct hws_cpu_buffer *cb;
863
864 cb = container_of(work, struct hws_cpu_buffer, worker);
865 cpu = smp_processor_id();
866 ext_params = atomic_xchg(&cb->ext_params, 0);
867
868 if (!cb->worker_entry)
869 worker_on_start(cpu);
870
871 if (worker_check_error(cpu, ext_params))
872 return;
873
874 if (!cb->finish)
875 worker_on_interrupt(cpu);
876
877 if (cb->finish)
878 worker_on_finish(cpu);
879}
880
881/**
882 * hwsampler_allocate() - allocate memory for the hardware sampler
883 * @sdbt: number of SDBTs per online CPU (must be > 0)
884 * @sdb: number of SDBs per SDBT (minimum 1, maximum 511)
885 *
886 * Returns 0 on success, !0 on failure.
887 */
888int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
889{
890 int cpu, rc;
891 mutex_lock(&hws_sem);
892
893 rc = -EINVAL;
894 if (hws_state != HWS_DEALLOCATED)
895 goto allocate_exit;
896
897 if (sdbt < 1)
898 goto allocate_exit;
899
900 if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
901 goto allocate_exit;
902
903 num_sdbt = sdbt;
904 num_sdb = sdb;
905
906 oom_killer_was_active = 0;
907 register_oom_notifier(&hws_oom_notifier);
908
909 for_each_online_cpu(cpu) {
910 if (allocate_sdbt(cpu)) {
911 unregister_oom_notifier(&hws_oom_notifier);
912 goto allocate_error;
913 }
914 }
915 unregister_oom_notifier(&hws_oom_notifier);
916 if (oom_killer_was_active)
917 goto allocate_error;
918
919 hws_state = HWS_STOPPED;
920 rc = 0;
921
922allocate_exit:
923 mutex_unlock(&hws_sem);
924 return rc;
925
926allocate_error:
927 rc = -ENOMEM;
928 printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
929 goto allocate_exit;
930}
931
932/**
933 * hwsampler_deallocate() - deallocate hardware sampler memory
934 *
935 * Returns 0 on success, !0 on failure.
936 */
937int hwsampler_deallocate(void)
938{
939 int rc;
940
941 mutex_lock(&hws_sem);
942
943 rc = -EINVAL;
944 if (hws_state != HWS_STOPPED)
945 goto deallocate_exit;
946
947 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
948 hws_alert = 0;
949 deallocate_sdbt();
950
951 hws_state = HWS_DEALLOCATED;
952 rc = 0;
953
954deallocate_exit:
955 mutex_unlock(&hws_sem);
956
957 return rc;
958}
959
960unsigned long hwsampler_query_min_interval(void)
961{
962 return min_sampler_rate;
963}
964
965unsigned long hwsampler_query_max_interval(void)
966{
967 return max_sampler_rate;
968}
969
970unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
971{
972 struct hws_cpu_buffer *cb;
973
974 cb = &per_cpu(sampler_cpu_buffer, cpu);
975
976 return cb->sample_overflow;
977}
978
979int hwsampler_setup(void)
980{
981 int rc;
982 int cpu;
983 struct hws_cpu_buffer *cb;
984
985 mutex_lock(&hws_sem);
986
987 rc = -EINVAL;
988 if (hws_state)
989 goto setup_exit;
990
991 hws_state = HWS_INIT;
992
993 init_all_cpu_buffers();
994
995 rc = check_hardware_prerequisites();
996 if (rc)
997 goto setup_exit;
998
999 rc = check_qsi_on_setup();
1000 if (rc)
1001 goto setup_exit;
1002
1003 rc = -EINVAL;
1004 hws_wq = create_workqueue("hwsampler");
1005 if (!hws_wq)
1006 goto setup_exit;
1007
1008 register_cpu_notifier(&hws_cpu_notifier);
1009
1010 for_each_online_cpu(cpu) {
1011 cb = &per_cpu(sampler_cpu_buffer, cpu);
1012 INIT_WORK(&cb->worker, worker);
1013 rc = smp_ctl_qsi(cpu);
1014 WARN_ON(rc);
1015 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1016 if (min_sampler_rate) {
1017 printk(KERN_WARNING
1018 "hwsampler: different min sampler rate values.\n");
1019 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1020 min_sampler_rate =
1021 cb->qsi.min_sampl_rate;
1022 } else
1023 min_sampler_rate = cb->qsi.min_sampl_rate;
1024 }
1025 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1026 if (max_sampler_rate) {
1027 printk(KERN_WARNING
1028 "hwsampler: different max sampler rate values.\n");
1029 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1030 max_sampler_rate =
1031 cb->qsi.max_sampl_rate;
1032 } else
1033 max_sampler_rate = cb->qsi.max_sampl_rate;
1034 }
1035 }
1036 register_external_irq(EXT_IRQ_MEASURE_ALERT, hws_ext_handler);
1037
1038 hws_state = HWS_DEALLOCATED;
1039 rc = 0;
1040
1041setup_exit:
1042 mutex_unlock(&hws_sem);
1043 return rc;
1044}
1045
1046int hwsampler_shutdown(void)
1047{
1048 int rc;
1049
1050 mutex_lock(&hws_sem);
1051
1052 rc = -EINVAL;
1053 if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1054 mutex_unlock(&hws_sem);
1055
1056 if (hws_wq)
1057 flush_workqueue(hws_wq);
1058
1059 mutex_lock(&hws_sem);
1060
1061 if (hws_state == HWS_STOPPED) {
1062 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
1063 hws_alert = 0;
1064 deallocate_sdbt();
1065 }
1066 if (hws_wq) {
1067 destroy_workqueue(hws_wq);
1068 hws_wq = NULL;
1069 }
1070
1071 unregister_external_irq(EXT_IRQ_MEASURE_ALERT, hws_ext_handler);
1072 hws_state = HWS_INIT;
1073 rc = 0;
1074 }
1075 mutex_unlock(&hws_sem);
1076
1077 unregister_cpu_notifier(&hws_cpu_notifier);
1078
1079 return rc;
1080}
1081
1082/**
1083 * hwsampler_start_all() - start hardware sampling on all online CPUs
1084 * @rate: specifies the used interval when samples are taken
1085 *
1086 * Returns 0 on success, !0 on failure.
1087 */
1088int hwsampler_start_all(unsigned long rate)
1089{
1090 int rc, cpu;
1091
1092 mutex_lock(&hws_sem);
1093
1094 hws_oom = 0;
1095
1096 rc = -EINVAL;
1097 if (hws_state != HWS_STOPPED)
1098 goto start_all_exit;
1099
1100 interval = rate;
1101
1102 /* fail if rate is not valid */
1103 if (interval < min_sampler_rate || interval > max_sampler_rate)
1104 goto start_all_exit;
1105
1106 rc = check_qsi_on_start();
1107 if (rc)
1108 goto start_all_exit;
1109
1110 rc = prepare_cpu_buffers();
1111 if (rc)
1112 goto start_all_exit;
1113
1114 for_each_online_cpu(cpu) {
1115 rc = start_sampling(cpu);
1116 if (rc)
1117 break;
1118 }
1119 if (rc) {
1120 for_each_online_cpu(cpu) {
1121 stop_sampling(cpu);
1122 }
1123 goto start_all_exit;
1124 }
1125 hws_state = HWS_STARTED;
1126 rc = 0;
1127
1128start_all_exit:
1129 mutex_unlock(&hws_sem);
1130
1131 if (rc)
1132 return rc;
1133
1134 register_oom_notifier(&hws_oom_notifier);
1135 hws_oom = 1;
1136 hws_flush_all = 0;
1137 /* now let them in, 1407 CPUMF external interrupts */
1138 hws_alert = 1;
1139 irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
1140
1141 return 0;
1142}
1143
1144/**
1145 * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1146 *
1147 * Returns 0 on success, !0 on failure.
1148 */
1149int hwsampler_stop_all(void)
1150{
1151 int tmp_rc, rc, cpu;
1152 struct hws_cpu_buffer *cb;
1153
1154 mutex_lock(&hws_sem);
1155
1156 rc = 0;
1157 if (hws_state == HWS_INIT) {
1158 mutex_unlock(&hws_sem);
1159 return rc;
1160 }
1161 hws_state = HWS_STOPPING;
1162 mutex_unlock(&hws_sem);
1163
1164 for_each_online_cpu(cpu) {
1165 cb = &per_cpu(sampler_cpu_buffer, cpu);
1166 cb->stop_mode = 1;
1167 tmp_rc = stop_sampling(cpu);
1168 if (tmp_rc)
1169 rc = tmp_rc;
1170 }
1171
1172 if (hws_wq)
1173 flush_workqueue(hws_wq);
1174
1175 mutex_lock(&hws_sem);
1176 if (hws_oom) {
1177 unregister_oom_notifier(&hws_oom_notifier);
1178 hws_oom = 0;
1179 }
1180 hws_state = HWS_STOPPED;
1181 mutex_unlock(&hws_sem);
1182
1183 return rc;
1184}