1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (c) 2022 Amlogic, Inc. All rights reserved.
  4 */
  5
  6#include <linux/bitfield.h>
  7#include <linux/init.h>
  8#include <linux/irqreturn.h>
  9#include <linux/kernel.h>
 10#include <linux/module.h>
 11#include <linux/of.h>
 12#include <linux/of_device.h>
 13#include <linux/of_irq.h>
 14#include <linux/perf_event.h>
 15#include <linux/platform_device.h>
 16#include <linux/printk.h>
 17#include <linux/sysfs.h>
 18#include <linux/types.h>
 19
 20#include <soc/amlogic/meson_ddr_pmu.h>
 21
 22struct ddr_pmu {
 23	struct pmu pmu;
 24	struct dmc_info info;
 25	struct dmc_counter counters;	/* save counters from hw */
 26	bool pmu_enabled;
 27	struct device *dev;
 28	char *name;
 29	struct hlist_node node;
 30	enum cpuhp_state cpuhp_state;
 31	int cpu;			/* for cpu hotplug */
 32};
 33
 34#define DDR_PERF_DEV_NAME "meson_ddr_bw"
 35#define MAX_AXI_PORTS_OF_CHANNEL	4	/* A DMC channel can monitor max 4 axi ports */
 36
 37#define to_ddr_pmu(p)		container_of(p, struct ddr_pmu, pmu)
 38#define dmc_info_to_pmu(p)	container_of(p, struct ddr_pmu, info)
 39
 40static void dmc_pmu_enable(struct ddr_pmu *pmu)
 41{
 42	if (!pmu->pmu_enabled)
 43		pmu->info.hw_info->enable(&pmu->info);
 44
 45	pmu->pmu_enabled = true;
 46}
 47
 48static void dmc_pmu_disable(struct ddr_pmu *pmu)
 49{
 50	if (pmu->pmu_enabled)
 51		pmu->info.hw_info->disable(&pmu->info);
 52
 53	pmu->pmu_enabled = false;
 54}
 55
 56static void meson_ddr_set_axi_filter(struct perf_event *event, u8 axi_id)
 57{
 58	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 59	int chann;
 60
 61	if (event->attr.config > ALL_CHAN_COUNTER_ID &&
 62	    event->attr.config < COUNTER_MAX_ID) {
 63		chann = event->attr.config - CHAN1_COUNTER_ID;
 64
 65		pmu->info.hw_info->set_axi_filter(&pmu->info, axi_id, chann);
 66	}
 67}
 68
 69static void ddr_cnt_addition(struct dmc_counter *sum,
 70			     struct dmc_counter *add1,
 71			     struct dmc_counter *add2,
 72			     int chann_nr)
 73{
 74	int i;
 75	u64 cnt1, cnt2;
 76
 77	sum->all_cnt = add1->all_cnt + add2->all_cnt;
 78	sum->all_req = add1->all_req + add2->all_req;
 79	for (i = 0; i < chann_nr; i++) {
 80		cnt1 = add1->channel_cnt[i];
 81		cnt2 = add2->channel_cnt[i];
 82
 83		sum->channel_cnt[i] = cnt1 + cnt2;
 84	}
 85}
 86
 87static void meson_ddr_perf_event_update(struct perf_event *event)
 88{
 89	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 90	u64 new_raw_count = 0;
 91	struct dmc_counter dc = {0}, sum_dc = {0};
 92	int idx;
 93	int chann_nr = pmu->info.hw_info->chann_nr;
 94
 95	/* get the remain counters in register. */
 96	pmu->info.hw_info->get_counters(&pmu->info, &dc);
 97
 98	ddr_cnt_addition(&sum_dc, &pmu->counters, &dc, chann_nr);
 99
100	switch (event->attr.config) {
101	case ALL_CHAN_COUNTER_ID:
102		new_raw_count = sum_dc.all_cnt;
103		break;
104	case CHAN1_COUNTER_ID:
105	case CHAN2_COUNTER_ID:
106	case CHAN3_COUNTER_ID:
107	case CHAN4_COUNTER_ID:
108	case CHAN5_COUNTER_ID:
109	case CHAN6_COUNTER_ID:
110	case CHAN7_COUNTER_ID:
111	case CHAN8_COUNTER_ID:
112		idx = event->attr.config - CHAN1_COUNTER_ID;
113		new_raw_count = sum_dc.channel_cnt[idx];
114		break;
115	}
116
117	local64_set(&event->count, new_raw_count);
118}
119
120static int meson_ddr_perf_event_init(struct perf_event *event)
121{
122	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
123	u64 config1 = event->attr.config1;
124	u64 config2 = event->attr.config2;
125
126	if (event->attr.type != event->pmu->type)
127		return -ENOENT;
128
129	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
130		return -EOPNOTSUPP;
131
132	if (event->cpu < 0)
133		return -EOPNOTSUPP;
134
135	/* check if the number of parameters is too much */
136	if (event->attr.config != ALL_CHAN_COUNTER_ID &&
137	    hweight64(config1) + hweight64(config2) > MAX_AXI_PORTS_OF_CHANNEL)
138		return -EOPNOTSUPP;
139
140	event->cpu = pmu->cpu;
141
142	return 0;
143}
144
145static void meson_ddr_perf_event_start(struct perf_event *event, int flags)
146{
147	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
148
149	memset(&pmu->counters, 0, sizeof(pmu->counters));
150	dmc_pmu_enable(pmu);
151}
152
153static int meson_ddr_perf_event_add(struct perf_event *event, int flags)
154{
155	u64 config1 = event->attr.config1;
156	u64 config2 = event->attr.config2;
157	int i;
158
159	for_each_set_bit(i, (const unsigned long *)&config1, sizeof(config1))
160		meson_ddr_set_axi_filter(event, i);
161
162	for_each_set_bit(i, (const unsigned long *)&config2, sizeof(config2))
163		meson_ddr_set_axi_filter(event, i + 64);
164
165	if (flags & PERF_EF_START)
166		meson_ddr_perf_event_start(event, flags);
167
168	return 0;
169}
170
171static void meson_ddr_perf_event_stop(struct perf_event *event, int flags)
172{
173	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
174
175	if (flags & PERF_EF_UPDATE)
176		meson_ddr_perf_event_update(event);
177
178	dmc_pmu_disable(pmu);
179}
180
181static void meson_ddr_perf_event_del(struct perf_event *event, int flags)
182{
183	meson_ddr_perf_event_stop(event, PERF_EF_UPDATE);
184}
185
186static ssize_t meson_ddr_perf_cpumask_show(struct device *dev,
187					   struct device_attribute *attr,
188					   char *buf)
189{
190	struct ddr_pmu *pmu = dev_get_drvdata(dev);
191
192	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
193}
194
195static struct device_attribute meson_ddr_perf_cpumask_attr =
196__ATTR(cpumask, 0444, meson_ddr_perf_cpumask_show, NULL);
197
198static struct attribute *meson_ddr_perf_cpumask_attrs[] = {
199	&meson_ddr_perf_cpumask_attr.attr,
200	NULL,
201};
202
203static const struct attribute_group ddr_perf_cpumask_attr_group = {
204	.attrs = meson_ddr_perf_cpumask_attrs,
205};
206
207static ssize_t
208pmu_event_show(struct device *dev, struct device_attribute *attr,
209	       char *page)
210{
211	struct perf_pmu_events_attr *pmu_attr;
212
213	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
214	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
215}
216
217static ssize_t
218event_show_unit(struct device *dev, struct device_attribute *attr,
219		char *page)
220{
221	return sysfs_emit(page, "MB\n");
222}
223
224static ssize_t
225event_show_scale(struct device *dev, struct device_attribute *attr,
226		 char *page)
227{
228	/* one count = 16byte = 1.52587890625e-05 MB */
229	return sysfs_emit(page, "1.52587890625e-05\n");
230}
231
232#define AML_DDR_PMU_EVENT_ATTR(_name, _id)				\
233{									\
234	.attr = __ATTR(_name, 0444, pmu_event_show, NULL),		\
235	.id = _id,							\
236}
237
238#define AML_DDR_PMU_EVENT_UNIT_ATTR(_name)				\
239	__ATTR(_name.unit, 0444, event_show_unit, NULL)
240
241#define AML_DDR_PMU_EVENT_SCALE_ATTR(_name)				\
242	__ATTR(_name.scale, 0444, event_show_scale, NULL)
243
244static struct device_attribute event_unit_attrs[] = {
245	AML_DDR_PMU_EVENT_UNIT_ATTR(total_rw_bytes),
246	AML_DDR_PMU_EVENT_UNIT_ATTR(chan_1_rw_bytes),
247	AML_DDR_PMU_EVENT_UNIT_ATTR(chan_2_rw_bytes),
248	AML_DDR_PMU_EVENT_UNIT_ATTR(chan_3_rw_bytes),
249	AML_DDR_PMU_EVENT_UNIT_ATTR(chan_4_rw_bytes),
250	AML_DDR_PMU_EVENT_UNIT_ATTR(chan_5_rw_bytes),
251	AML_DDR_PMU_EVENT_UNIT_ATTR(chan_6_rw_bytes),
252	AML_DDR_PMU_EVENT_UNIT_ATTR(chan_7_rw_bytes),
253	AML_DDR_PMU_EVENT_UNIT_ATTR(chan_8_rw_bytes),
254};
255
256static struct device_attribute event_scale_attrs[] = {
257	AML_DDR_PMU_EVENT_SCALE_ATTR(total_rw_bytes),
258	AML_DDR_PMU_EVENT_SCALE_ATTR(chan_1_rw_bytes),
259	AML_DDR_PMU_EVENT_SCALE_ATTR(chan_2_rw_bytes),
260	AML_DDR_PMU_EVENT_SCALE_ATTR(chan_3_rw_bytes),
261	AML_DDR_PMU_EVENT_SCALE_ATTR(chan_4_rw_bytes),
262	AML_DDR_PMU_EVENT_SCALE_ATTR(chan_5_rw_bytes),
263	AML_DDR_PMU_EVENT_SCALE_ATTR(chan_6_rw_bytes),
264	AML_DDR_PMU_EVENT_SCALE_ATTR(chan_7_rw_bytes),
265	AML_DDR_PMU_EVENT_SCALE_ATTR(chan_8_rw_bytes),
266};
267
268static struct perf_pmu_events_attr event_attrs[] = {
269	AML_DDR_PMU_EVENT_ATTR(total_rw_bytes, ALL_CHAN_COUNTER_ID),
270	AML_DDR_PMU_EVENT_ATTR(chan_1_rw_bytes, CHAN1_COUNTER_ID),
271	AML_DDR_PMU_EVENT_ATTR(chan_2_rw_bytes, CHAN2_COUNTER_ID),
272	AML_DDR_PMU_EVENT_ATTR(chan_3_rw_bytes, CHAN3_COUNTER_ID),
273	AML_DDR_PMU_EVENT_ATTR(chan_4_rw_bytes, CHAN4_COUNTER_ID),
274	AML_DDR_PMU_EVENT_ATTR(chan_5_rw_bytes, CHAN5_COUNTER_ID),
275	AML_DDR_PMU_EVENT_ATTR(chan_6_rw_bytes, CHAN6_COUNTER_ID),
276	AML_DDR_PMU_EVENT_ATTR(chan_7_rw_bytes, CHAN7_COUNTER_ID),
277	AML_DDR_PMU_EVENT_ATTR(chan_8_rw_bytes, CHAN8_COUNTER_ID),
278};
279
280/* three attrs are combined an event */
281static struct attribute *ddr_perf_events_attrs[COUNTER_MAX_ID * 3];
282
283static struct attribute_group ddr_perf_events_attr_group = {
284	.name = "events",
285	.attrs = ddr_perf_events_attrs,
286};
287
288static umode_t meson_ddr_perf_format_attr_visible(struct kobject *kobj,
289						  struct attribute *attr,
290						  int n)
291{
292	struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
293	struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
294	const u64 *capability = ddr_pmu->info.hw_info->capability;
295	struct device_attribute *dev_attr;
296	int id;
297	char value[20]; // config1:xxx, 20 is enough
298
299	dev_attr = container_of(attr, struct device_attribute, attr);
300	dev_attr->show(NULL, NULL, value);
301
302	if (sscanf(value, "config1:%d", &id) == 1)
303		return capability[0] & (1ULL << id) ? attr->mode : 0;
304
305	if (sscanf(value, "config2:%d", &id) == 1)
306		return capability[1] & (1ULL << id) ? attr->mode : 0;
307
308	return attr->mode;
309}
310
311static struct attribute_group ddr_perf_format_attr_group = {
312	.name = "format",
313	.is_visible = meson_ddr_perf_format_attr_visible,
314};
315
316static ssize_t meson_ddr_perf_identifier_show(struct device *dev,
317					      struct device_attribute *attr,
318					      char *page)
319{
320	struct ddr_pmu *pmu = dev_get_drvdata(dev);
321
322	return sysfs_emit(page, "%s\n", pmu->name);
323}
324
325static struct device_attribute meson_ddr_perf_identifier_attr =
326__ATTR(identifier, 0444, meson_ddr_perf_identifier_show, NULL);
327
328static struct attribute *meson_ddr_perf_identifier_attrs[] = {
329	&meson_ddr_perf_identifier_attr.attr,
330	NULL,
331};
332
333static const struct attribute_group ddr_perf_identifier_attr_group = {
334	.attrs = meson_ddr_perf_identifier_attrs,
335};
336
337static const struct attribute_group *attr_groups[] = {
338	&ddr_perf_events_attr_group,
339	&ddr_perf_format_attr_group,
340	&ddr_perf_cpumask_attr_group,
341	&ddr_perf_identifier_attr_group,
342	NULL,
343};
344
345static irqreturn_t dmc_irq_handler(int irq, void *dev_id)
346{
347	struct dmc_info *info = dev_id;
348	struct ddr_pmu *pmu;
349	struct dmc_counter counters, *sum_cnter;
350	int i;
351
352	pmu = dmc_info_to_pmu(info);
353
354	if (info->hw_info->irq_handler(info, &counters) != 0)
355		goto out;
356
357	sum_cnter = &pmu->counters;
358	sum_cnter->all_cnt += counters.all_cnt;
359	sum_cnter->all_req += counters.all_req;
360
361	for (i = 0; i < pmu->info.hw_info->chann_nr; i++)
362		sum_cnter->channel_cnt[i] += counters.channel_cnt[i];
363
364	if (pmu->pmu_enabled)
365		/*
366		 * the timer interrupt only supprt
367		 * one shot mode, we have to re-enable
368		 * it in ISR to support continue mode.
369		 */
370		info->hw_info->enable(info);
371
372	dev_dbg(pmu->dev, "counts: %llu %llu %llu, %llu, %llu, %llu\t\t"
373			"sum: %llu %llu %llu, %llu, %llu, %llu\n",
374			counters.all_req,
375			counters.all_cnt,
376			counters.channel_cnt[0],
377			counters.channel_cnt[1],
378			counters.channel_cnt[2],
379			counters.channel_cnt[3],
380
381			pmu->counters.all_req,
382			pmu->counters.all_cnt,
383			pmu->counters.channel_cnt[0],
384			pmu->counters.channel_cnt[1],
385			pmu->counters.channel_cnt[2],
386			pmu->counters.channel_cnt[3]);
387out:
388	return IRQ_HANDLED;
389}
390
391static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
392{
393	struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
394	int target;
395
396	if (cpu != pmu->cpu)
397		return 0;
398
399	target = cpumask_any_but(cpu_online_mask, cpu);
400	if (target >= nr_cpu_ids)
401		return 0;
402
403	perf_pmu_migrate_context(&pmu->pmu, cpu, target);
404	pmu->cpu = target;
405
406	WARN_ON(irq_set_affinity(pmu->info.irq_num, cpumask_of(pmu->cpu)));
407
408	return 0;
409}
410
411static void fill_event_attr(struct ddr_pmu *pmu)
412{
413	int i, j, k;
414	struct attribute **dst = ddr_perf_events_attrs;
415
416	j = 0;
417	k = 0;
418
419	/* fill ALL_CHAN_COUNTER_ID event */
420	dst[j++] = &event_attrs[k].attr.attr;
421	dst[j++] = &event_unit_attrs[k].attr;
422	dst[j++] = &event_scale_attrs[k].attr;
423
424	k++;
425
426	/* fill each channel event */
427	for (i = 0; i < pmu->info.hw_info->chann_nr; i++, k++) {
428		dst[j++] = &event_attrs[k].attr.attr;
429		dst[j++] = &event_unit_attrs[k].attr;
430		dst[j++] = &event_scale_attrs[k].attr;
431	}
432
433	dst[j] = NULL; /* mark end */
434}
435
436static void fmt_attr_fill(struct attribute **fmt_attr)
437{
438	ddr_perf_format_attr_group.attrs = fmt_attr;
439}
440
441static int ddr_pmu_parse_dt(struct platform_device *pdev,
442			    struct dmc_info *info)
443{
444	void __iomem *base;
445	int i, ret;
446
447	info->hw_info = of_device_get_match_data(&pdev->dev);
448
449	for (i = 0; i < info->hw_info->dmc_nr; i++) {
450		/* resource 0 for ddr register base */
451		base = devm_platform_ioremap_resource(pdev, i);
452		if (IS_ERR(base))
453			return PTR_ERR(base);
454
455		info->ddr_reg[i] = base;
456	}
457
458	/* resource i for pll register base */
459	base = devm_platform_ioremap_resource(pdev, i);
460	if (IS_ERR(base))
461		return PTR_ERR(base);
462
463	info->pll_reg = base;
464
465	ret = platform_get_irq(pdev, 0);
466	if (ret < 0)
467		return ret;
468
469	info->irq_num = ret;
470
471	ret = devm_request_irq(&pdev->dev, info->irq_num, dmc_irq_handler,
472			       IRQF_NOBALANCING, dev_name(&pdev->dev),
473			       (void *)info);
474	if (ret < 0)
475		return ret;
476
477	return 0;
478}
479
480int meson_ddr_pmu_create(struct platform_device *pdev)
481{
482	int ret;
483	char *name;
484	struct ddr_pmu *pmu;
485
486	pmu = devm_kzalloc(&pdev->dev, sizeof(struct ddr_pmu), GFP_KERNEL);
487	if (!pmu)
488		return -ENOMEM;
489
490	*pmu = (struct ddr_pmu) {
491		.pmu = {
492			.module		= THIS_MODULE,
493			.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
494			.task_ctx_nr	= perf_invalid_context,
495			.attr_groups	= attr_groups,
496			.event_init	= meson_ddr_perf_event_init,
497			.add		= meson_ddr_perf_event_add,
498			.del		= meson_ddr_perf_event_del,
499			.start		= meson_ddr_perf_event_start,
500			.stop		= meson_ddr_perf_event_stop,
501			.read		= meson_ddr_perf_event_update,
502		},
503	};
504
505	ret = ddr_pmu_parse_dt(pdev, &pmu->info);
506	if (ret < 0)
507		return ret;
508
509	fmt_attr_fill(pmu->info.hw_info->fmt_attr);
510
511	pmu->cpu = smp_processor_id();
512
513	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME);
514	if (!name)
515		return -ENOMEM;
516
517	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, name, NULL,
518				      ddr_perf_offline_cpu);
519	if (ret < 0)
520		return ret;
521
522	pmu->cpuhp_state = ret;
523
524	/* Register the pmu instance for cpu hotplug */
525	ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
526	if (ret)
527		goto cpuhp_instance_err;
528
529	fill_event_attr(pmu);
530
531	ret = perf_pmu_register(&pmu->pmu, name, -1);
532	if (ret)
533		goto pmu_register_err;
534
535	pmu->name = name;
536	pmu->dev = &pdev->dev;
537	pmu->pmu_enabled = false;
538
539	platform_set_drvdata(pdev, pmu);
540
541	return 0;
542
543pmu_register_err:
544	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
545
546cpuhp_instance_err:
547	cpuhp_remove_state(pmu->cpuhp_state);
548
549	return ret;
550}
551
552int meson_ddr_pmu_remove(struct platform_device *pdev)
553{
554	struct ddr_pmu *pmu = platform_get_drvdata(pdev);
555
556	perf_pmu_unregister(&pmu->pmu);
557	cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
558	cpuhp_remove_state(pmu->cpuhp_state);
559
560	return 0;
561}