Loading...
Note: File does not exist in v4.10.11.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2019 Samsung Electronics Co., Ltd.
4 * Author: Lukasz Luba <l.luba@partner.samsung.com>
5 */
6
7#include <linux/clk.h>
8#include <linux/devfreq.h>
9#include <linux/devfreq-event.h>
10#include <linux/device.h>
11#include <linux/interrupt.h>
12#include <linux/io.h>
13#include <linux/mfd/syscon.h>
14#include <linux/module.h>
15#include <linux/moduleparam.h>
16#include <linux/of_device.h>
17#include <linux/pm_opp.h>
18#include <linux/platform_device.h>
19#include <linux/regmap.h>
20#include <linux/regulator/consumer.h>
21#include <linux/slab.h>
22#include "../jedec_ddr.h"
23#include "../of_memory.h"
24
25static int irqmode;
26module_param(irqmode, int, 0644);
27MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)");
28
29#define EXYNOS5_DREXI_TIMINGAREF (0x0030)
30#define EXYNOS5_DREXI_TIMINGROW0 (0x0034)
31#define EXYNOS5_DREXI_TIMINGDATA0 (0x0038)
32#define EXYNOS5_DREXI_TIMINGPOWER0 (0x003C)
33#define EXYNOS5_DREXI_TIMINGROW1 (0x00E4)
34#define EXYNOS5_DREXI_TIMINGDATA1 (0x00E8)
35#define EXYNOS5_DREXI_TIMINGPOWER1 (0x00EC)
36#define CDREX_PAUSE (0x2091c)
37#define CDREX_LPDDR3PHY_CON3 (0x20a20)
38#define CDREX_LPDDR3PHY_CLKM_SRC (0x20700)
39#define EXYNOS5_TIMING_SET_SWI BIT(28)
40#define USE_MX_MSPLL_TIMINGS (1)
41#define USE_BPLL_TIMINGS (0)
42#define EXYNOS5_AREF_NORMAL (0x2e)
43
44#define DREX_PPCCLKCON (0x0130)
45#define DREX_PEREV2CONFIG (0x013c)
46#define DREX_PMNC_PPC (0xE000)
47#define DREX_CNTENS_PPC (0xE010)
48#define DREX_CNTENC_PPC (0xE020)
49#define DREX_INTENS_PPC (0xE030)
50#define DREX_INTENC_PPC (0xE040)
51#define DREX_FLAG_PPC (0xE050)
52#define DREX_PMCNT2_PPC (0xE130)
53
54/*
55 * A value for register DREX_PMNC_PPC which should be written to reset
56 * the cycle counter CCNT (a reference wall clock). It sets zero to the
57 * CCNT counter.
58 */
59#define CC_RESET BIT(2)
60
61/*
62 * A value for register DREX_PMNC_PPC which does the reset of all performance
63 * counters to zero.
64 */
65#define PPC_COUNTER_RESET BIT(1)
66
67/*
68 * Enables all configured counters (including cycle counter). The value should
69 * be written to the register DREX_PMNC_PPC.
70 */
71#define PPC_ENABLE BIT(0)
72
73/* A value for register DREX_PPCCLKCON which enables performance events clock.
74 * Must be written before first access to the performance counters register
75 * set, otherwise it could crash.
76 */
77#define PEREV_CLK_EN BIT(0)
78
79/*
80 * Values which are used to enable counters, interrupts or configure flags of
81 * the performance counters. They configure counter 2 and cycle counter.
82 */
83#define PERF_CNT2 BIT(2)
84#define PERF_CCNT BIT(31)
85
86/*
87 * Performance event types which are used for setting the preferred event
88 * to track in the counters.
89 * There is a set of different types, the values are from range 0 to 0x6f.
90 * These settings should be written to the configuration register which manages
91 * the type of the event (register DREX_PEREV2CONFIG).
92 */
93#define READ_TRANSFER_CH0 (0x6d)
94#define READ_TRANSFER_CH1 (0x6f)
95
96#define PERF_COUNTER_START_VALUE 0xff000000
97#define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL
98
99/**
100 * struct dmc_opp_table - Operating level desciption
101 *
102 * Covers frequency and voltage settings of the DMC operating mode.
103 */
104struct dmc_opp_table {
105 u32 freq_hz;
106 u32 volt_uv;
107};
108
109/**
110 * struct exynos5_dmc - main structure describing DMC device
111 *
112 * The main structure for the Dynamic Memory Controller which covers clocks,
113 * memory regions, HW information, parameters and current operating mode.
114 */
115struct exynos5_dmc {
116 struct device *dev;
117 struct devfreq *df;
118 struct devfreq_simple_ondemand_data gov_data;
119 void __iomem *base_drexi0;
120 void __iomem *base_drexi1;
121 struct regmap *clk_regmap;
122 struct mutex lock;
123 unsigned long curr_rate;
124 unsigned long curr_volt;
125 unsigned long bypass_rate;
126 struct dmc_opp_table *opp;
127 struct dmc_opp_table opp_bypass;
128 int opp_count;
129 u32 timings_arr_size;
130 u32 *timing_row;
131 u32 *timing_data;
132 u32 *timing_power;
133 const struct lpddr3_timings *timings;
134 const struct lpddr3_min_tck *min_tck;
135 u32 bypass_timing_row;
136 u32 bypass_timing_data;
137 u32 bypass_timing_power;
138 struct regulator *vdd_mif;
139 struct clk *fout_spll;
140 struct clk *fout_bpll;
141 struct clk *mout_spll;
142 struct clk *mout_bpll;
143 struct clk *mout_mclk_cdrex;
144 struct clk *mout_mx_mspll_ccore;
145 struct clk *mx_mspll_ccore_phy;
146 struct clk *mout_mx_mspll_ccore_phy;
147 struct devfreq_event_dev **counter;
148 int num_counters;
149 u64 last_overflow_ts[2];
150 unsigned long load;
151 unsigned long total;
152 bool in_irq_mode;
153};
154
155#define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \
156 { .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end }
157
158#define TIMING_VAL2REG(timing, t_val) \
159({ \
160 u32 __val; \
161 __val = (t_val) << (timing)->bit_beg; \
162 __val; \
163})
164
165struct timing_reg {
166 char *name;
167 int bit_beg;
168 int bit_end;
169 unsigned int val;
170};
171
172static const struct timing_reg timing_row[] = {
173 TIMING_FIELD("tRFC", 24, 31),
174 TIMING_FIELD("tRRD", 20, 23),
175 TIMING_FIELD("tRP", 16, 19),
176 TIMING_FIELD("tRCD", 12, 15),
177 TIMING_FIELD("tRC", 6, 11),
178 TIMING_FIELD("tRAS", 0, 5),
179};
180
181static const struct timing_reg timing_data[] = {
182 TIMING_FIELD("tWTR", 28, 31),
183 TIMING_FIELD("tWR", 24, 27),
184 TIMING_FIELD("tRTP", 20, 23),
185 TIMING_FIELD("tW2W-C2C", 14, 14),
186 TIMING_FIELD("tR2R-C2C", 12, 12),
187 TIMING_FIELD("WL", 8, 11),
188 TIMING_FIELD("tDQSCK", 4, 7),
189 TIMING_FIELD("RL", 0, 3),
190};
191
192static const struct timing_reg timing_power[] = {
193 TIMING_FIELD("tFAW", 26, 31),
194 TIMING_FIELD("tXSR", 16, 25),
195 TIMING_FIELD("tXP", 8, 15),
196 TIMING_FIELD("tCKE", 4, 7),
197 TIMING_FIELD("tMRD", 0, 3),
198};
199
200#define TIMING_COUNT (ARRAY_SIZE(timing_row) + ARRAY_SIZE(timing_data) + \
201 ARRAY_SIZE(timing_power))
202
203static int exynos5_counters_set_event(struct exynos5_dmc *dmc)
204{
205 int i, ret;
206
207 for (i = 0; i < dmc->num_counters; i++) {
208 if (!dmc->counter[i])
209 continue;
210 ret = devfreq_event_set_event(dmc->counter[i]);
211 if (ret < 0)
212 return ret;
213 }
214 return 0;
215}
216
217static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)
218{
219 int i, ret;
220
221 for (i = 0; i < dmc->num_counters; i++) {
222 if (!dmc->counter[i])
223 continue;
224 ret = devfreq_event_enable_edev(dmc->counter[i]);
225 if (ret < 0)
226 return ret;
227 }
228 return 0;
229}
230
231static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)
232{
233 int i, ret;
234
235 for (i = 0; i < dmc->num_counters; i++) {
236 if (!dmc->counter[i])
237 continue;
238 ret = devfreq_event_disable_edev(dmc->counter[i]);
239 if (ret < 0)
240 return ret;
241 }
242 return 0;
243}
244
245/**
246 * find_target_freq_id() - Finds requested frequency in local DMC configuration
247 * @dmc: device for which the information is checked
248 * @target_rate: requested frequency in KHz
249 *
250 * Seeks in the local DMC driver structure for the requested frequency value
251 * and returns index or error value.
252 */
253static int find_target_freq_idx(struct exynos5_dmc *dmc,
254 unsigned long target_rate)
255{
256 int i;
257
258 for (i = dmc->opp_count - 1; i >= 0; i--)
259 if (dmc->opp[i].freq_hz <= target_rate)
260 return i;
261
262 return -EINVAL;
263}
264
265/**
266 * exynos5_switch_timing_regs() - Changes bank register set for DRAM timings
267 * @dmc: device for which the new settings is going to be applied
268 * @set: boolean variable passing set value
269 *
270 * Changes the register set, which holds timing parameters.
271 * There is two register sets: 0 and 1. The register set 0
272 * is used in normal operation when the clock is provided from main PLL.
273 * The bank register set 1 is used when the main PLL frequency is going to be
274 * changed and the clock is taken from alternative, stable source.
275 * This function switches between these banks according to the
276 * currently used clock source.
277 */
278static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)
279{
280 unsigned int reg;
281 int ret;
282
283 ret = regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, ®);
284 if (ret)
285 return ret;
286
287 if (set)
288 reg |= EXYNOS5_TIMING_SET_SWI;
289 else
290 reg &= ~EXYNOS5_TIMING_SET_SWI;
291
292 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, reg);
293
294 return 0;
295}
296
297/**
298 * exynos5_init_freq_table() - Initialized PM OPP framework
299 * @dmc: DMC device for which the frequencies are used for OPP init
300 * @profile: devfreq device's profile
301 *
302 * Populate the devfreq device's OPP table based on current frequency, voltage.
303 */
304static int exynos5_init_freq_table(struct exynos5_dmc *dmc,
305 struct devfreq_dev_profile *profile)
306{
307 int i, ret;
308 int idx;
309 unsigned long freq;
310
311 ret = dev_pm_opp_of_add_table(dmc->dev);
312 if (ret < 0) {
313 dev_err(dmc->dev, "Failed to get OPP table\n");
314 return ret;
315 }
316
317 dmc->opp_count = dev_pm_opp_get_opp_count(dmc->dev);
318
319 dmc->opp = devm_kmalloc_array(dmc->dev, dmc->opp_count,
320 sizeof(struct dmc_opp_table), GFP_KERNEL);
321 if (!dmc->opp)
322 goto err_opp;
323
324 idx = dmc->opp_count - 1;
325 for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {
326 struct dev_pm_opp *opp;
327
328 opp = dev_pm_opp_find_freq_floor(dmc->dev, &freq);
329 if (IS_ERR(opp))
330 goto err_opp;
331
332 dmc->opp[idx - i].freq_hz = freq;
333 dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);
334
335 dev_pm_opp_put(opp);
336 }
337
338 return 0;
339
340err_opp:
341 dev_pm_opp_of_remove_table(dmc->dev);
342
343 return -EINVAL;
344}
345
346/**
347 * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings
348 * @dmc: device for which the new settings is going to be applied
349 * @param: DRAM parameters which passes timing data
350 *
351 * Low-level function for changing timings for DRAM memory clocking from
352 * 'bypass' clock source (fixed frequency @400MHz).
353 * It uses timing bank registers set 1.
354 */
355static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)
356{
357 writel(EXYNOS5_AREF_NORMAL,
358 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
359
360 writel(dmc->bypass_timing_row,
361 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);
362 writel(dmc->bypass_timing_row,
363 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);
364 writel(dmc->bypass_timing_data,
365 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);
366 writel(dmc->bypass_timing_data,
367 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);
368 writel(dmc->bypass_timing_power,
369 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);
370 writel(dmc->bypass_timing_power,
371 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);
372}
373
374/**
375 * exynos5_dram_change_timings() - Low-level changes of the DRAM final timings
376 * @dmc: device for which the new settings is going to be applied
377 * @target_rate: target frequency of the DMC
378 *
379 * Low-level function for changing timings for DRAM memory operating from main
380 * clock source (BPLL), which can have different frequencies. Thus, each
381 * frequency must have corresponding timings register values in order to keep
382 * the needed delays.
383 * It uses timing bank registers set 0.
384 */
385static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,
386 unsigned long target_rate)
387{
388 int idx;
389
390 for (idx = dmc->opp_count - 1; idx >= 0; idx--)
391 if (dmc->opp[idx].freq_hz <= target_rate)
392 break;
393
394 if (idx < 0)
395 return -EINVAL;
396
397 writel(EXYNOS5_AREF_NORMAL,
398 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
399
400 writel(dmc->timing_row[idx],
401 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);
402 writel(dmc->timing_row[idx],
403 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);
404 writel(dmc->timing_data[idx],
405 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);
406 writel(dmc->timing_data[idx],
407 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);
408 writel(dmc->timing_power[idx],
409 dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);
410 writel(dmc->timing_power[idx],
411 dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);
412
413 return 0;
414}
415
416/**
417 * exynos5_dmc_align_target_voltage() - Sets the final voltage for the DMC
418 * @dmc: device for which it is going to be set
419 * @target_volt: new voltage which is chosen to be final
420 *
421 * Function tries to align voltage to the safe level for 'normal' mode.
422 * It checks the need of higher voltage and changes the value. The target
423 * voltage might be lower that currently set and still the system will be
424 * stable.
425 */
426static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,
427 unsigned long target_volt)
428{
429 int ret = 0;
430
431 if (dmc->curr_volt <= target_volt)
432 return 0;
433
434 ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
435 target_volt);
436 if (!ret)
437 dmc->curr_volt = target_volt;
438
439 return ret;
440}
441
442/**
443 * exynos5_dmc_align_bypass_voltage() - Sets the voltage for the DMC
444 * @dmc: device for which it is going to be set
445 * @target_volt: new voltage which is chosen to be final
446 *
447 * Function tries to align voltage to the safe level for the 'bypass' mode.
448 * It checks the need of higher voltage and changes the value.
449 * The target voltage must not be less than currently needed, because
450 * for current frequency the device might become unstable.
451 */
452static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,
453 unsigned long target_volt)
454{
455 int ret = 0;
456 unsigned long bypass_volt = dmc->opp_bypass.volt_uv;
457
458 target_volt = max(bypass_volt, target_volt);
459
460 if (dmc->curr_volt >= target_volt)
461 return 0;
462
463 ret = regulator_set_voltage(dmc->vdd_mif, target_volt,
464 target_volt);
465 if (!ret)
466 dmc->curr_volt = target_volt;
467
468 return ret;
469}
470
471/**
472 * exynos5_dmc_align_bypass_dram_timings() - Chooses and sets DRAM timings
473 * @dmc: device for which it is going to be set
474 * @target_rate: new frequency which is chosen to be final
475 *
476 * Function changes the DRAM timings for the temporary 'bypass' mode.
477 */
478static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,
479 unsigned long target_rate)
480{
481 int idx = find_target_freq_idx(dmc, target_rate);
482
483 if (idx < 0)
484 return -EINVAL;
485
486 exynos5_set_bypass_dram_timings(dmc);
487
488 return 0;
489}
490
491/**
492 * exynos5_dmc_switch_to_bypass_configuration() - Switching to temporary clock
493 * @dmc: DMC device for which the switching is going to happen
494 * @target_rate: new frequency which is going to be set as a final
495 * @target_volt: new voltage which is going to be set as a final
496 *
497 * Function configures DMC and clocks for operating in temporary 'bypass' mode.
498 * This mode is used only temporary but if required, changes voltage and timings
499 * for DRAM chips. It switches the main clock to stable clock source for the
500 * period of the main PLL reconfiguration.
501 */
502static int
503exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,
504 unsigned long target_rate,
505 unsigned long target_volt)
506{
507 int ret;
508
509 /*
510 * Having higher voltage for a particular frequency does not harm
511 * the chip. Use it for the temporary frequency change when one
512 * voltage manipulation might be avoided.
513 */
514 ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);
515 if (ret)
516 return ret;
517
518 /*
519 * Longer delays for DRAM does not cause crash, the opposite does.
520 */
521 ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);
522 if (ret)
523 return ret;
524
525 /*
526 * Delays are long enough, so use them for the new coming clock.
527 */
528 ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);
529
530 return ret;
531}
532
533/**
534 * exynos5_dmc_change_freq_and_volt() - Changes voltage and frequency of the DMC
535 * using safe procedure
536 * @dmc: device for which the frequency is going to be changed
537 * @target_rate: requested new frequency
538 * @target_volt: requested voltage which corresponds to the new frequency
539 *
540 * The DMC frequency change procedure requires a few steps.
541 * The main requirement is to change the clock source in the clk mux
542 * for the time of main clock PLL locking. The assumption is that the
543 * alternative clock source set as parent is stable.
544 * The second parent's clock frequency is fixed to 400MHz, it is named 'bypass'
545 * clock. This requires alignment in DRAM timing parameters for the new
546 * T-period. There is two bank sets for keeping DRAM
547 * timings: set 0 and set 1. The set 0 is used when main clock source is
548 * chosen. The 2nd set of regs is used for 'bypass' clock. Switching between
549 * the two bank sets is part of the process.
550 * The voltage must also be aligned to the minimum required level. There is
551 * this intermediate step with switching to 'bypass' parent clock source.
552 * if the old voltage is lower, it requires an increase of the voltage level.
553 * The complexity of the voltage manipulation is hidden in low level function.
554 * In this function there is last alignment of the voltage level at the end.
555 */
556static int
557exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,
558 unsigned long target_rate,
559 unsigned long target_volt)
560{
561 int ret;
562
563 ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,
564 target_volt);
565 if (ret)
566 return ret;
567
568 /*
569 * Voltage is set at least to a level needed for this frequency,
570 * so switching clock source is safe now.
571 */
572 clk_prepare_enable(dmc->fout_spll);
573 clk_prepare_enable(dmc->mout_spll);
574 clk_prepare_enable(dmc->mout_mx_mspll_ccore);
575
576 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_mx_mspll_ccore);
577 if (ret)
578 goto disable_clocks;
579
580 /*
581 * We are safe to increase the timings for current bypass frequency.
582 * Thanks to this the settings will be ready for the upcoming clock
583 * source change.
584 */
585 exynos5_dram_change_timings(dmc, target_rate);
586
587 clk_set_rate(dmc->fout_bpll, target_rate);
588
589 ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);
590 if (ret)
591 goto disable_clocks;
592
593 ret = clk_set_parent(dmc->mout_mclk_cdrex, dmc->mout_bpll);
594 if (ret)
595 goto disable_clocks;
596
597 /*
598 * Make sure if the voltage is not from 'bypass' settings and align to
599 * the right level for power efficiency.
600 */
601 ret = exynos5_dmc_align_target_voltage(dmc, target_volt);
602
603disable_clocks:
604 clk_disable_unprepare(dmc->mout_mx_mspll_ccore);
605 clk_disable_unprepare(dmc->mout_spll);
606 clk_disable_unprepare(dmc->fout_spll);
607
608 return ret;
609}
610
611/**
612 * exynos5_dmc_get_volt_freq() - Gets the frequency and voltage from the OPP
613 * table.
614 * @dmc: device for which the frequency is going to be changed
615 * @freq: requested frequency in KHz
616 * @target_rate: returned frequency which is the same or lower than
617 * requested
618 * @target_volt: returned voltage which corresponds to the returned
619 * frequency
620 *
621 * Function gets requested frequency and checks OPP framework for needed
622 * frequency and voltage. It populates the values 'target_rate' and
623 * 'target_volt' or returns error value when OPP framework fails.
624 */
625static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,
626 unsigned long *freq,
627 unsigned long *target_rate,
628 unsigned long *target_volt, u32 flags)
629{
630 struct dev_pm_opp *opp;
631
632 opp = devfreq_recommended_opp(dmc->dev, freq, flags);
633 if (IS_ERR(opp))
634 return PTR_ERR(opp);
635
636 *target_rate = dev_pm_opp_get_freq(opp);
637 *target_volt = dev_pm_opp_get_voltage(opp);
638 dev_pm_opp_put(opp);
639
640 return 0;
641}
642
643/**
644 * exynos5_dmc_target() - Function responsible for changing frequency of DMC
645 * @dev: device for which the frequency is going to be changed
646 * @freq: requested frequency in KHz
647 * @flags: flags provided for this frequency change request
648 *
649 * An entry function provided to the devfreq framework which provides frequency
650 * change of the DMC. The function gets the possible rate from OPP table based
651 * on requested frequency. It calls the next function responsible for the
652 * frequency and voltage change. In case of failure, does not set 'curr_rate'
653 * and returns error value to the framework.
654 */
655static int exynos5_dmc_target(struct device *dev, unsigned long *freq,
656 u32 flags)
657{
658 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
659 unsigned long target_rate = 0;
660 unsigned long target_volt = 0;
661 int ret;
662
663 ret = exynos5_dmc_get_volt_freq(dmc, freq, &target_rate, &target_volt,
664 flags);
665
666 if (ret)
667 return ret;
668
669 if (target_rate == dmc->curr_rate)
670 return 0;
671
672 mutex_lock(&dmc->lock);
673
674 ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);
675
676 if (ret) {
677 mutex_unlock(&dmc->lock);
678 return ret;
679 }
680
681 dmc->curr_rate = target_rate;
682
683 mutex_unlock(&dmc->lock);
684 return 0;
685}
686
687/**
688 * exynos5_counters_get() - Gets the performance counters values.
689 * @dmc: device for which the counters are going to be checked
690 * @load_count: variable which is populated with counter value
691 * @total_count: variable which is used as 'wall clock' reference
692 *
693 * Function which provides performance counters values. It sums up counters for
694 * two DMC channels. The 'total_count' is used as a reference and max value.
695 * The ratio 'load_count/total_count' shows the busy percentage [0%, 100%].
696 */
697static int exynos5_counters_get(struct exynos5_dmc *dmc,
698 unsigned long *load_count,
699 unsigned long *total_count)
700{
701 unsigned long total = 0;
702 struct devfreq_event_data event;
703 int ret, i;
704
705 *load_count = 0;
706
707 /* Take into account only read+write counters, but stop all */
708 for (i = 0; i < dmc->num_counters; i++) {
709 if (!dmc->counter[i])
710 continue;
711
712 ret = devfreq_event_get_event(dmc->counter[i], &event);
713 if (ret < 0)
714 return ret;
715
716 *load_count += event.load_count;
717
718 if (total < event.total_count)
719 total = event.total_count;
720 }
721
722 *total_count = total;
723
724 return 0;
725}
726
727/**
728 * exynos5_dmc_start_perf_events() - Setup and start performance event counters
729 * @dmc: device for which the counters are going to be checked
730 * @beg_value: initial value for the counter
731 *
732 * Function which enables needed counters, interrupts and sets initial values
733 * then starts the counters.
734 */
735static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,
736 u32 beg_value)
737{
738 /* Enable interrupts for counter 2 */
739 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENS_PPC);
740 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENS_PPC);
741
742 /* Enable counter 2 and CCNT */
743 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENS_PPC);
744 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENS_PPC);
745
746 /* Clear overflow flag for all counters */
747 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
748 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
749
750 /* Reset all counters */
751 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi0 + DREX_PMNC_PPC);
752 writel(CC_RESET | PPC_COUNTER_RESET, dmc->base_drexi1 + DREX_PMNC_PPC);
753
754 /*
755 * Set start value for the counters, the number of samples that
756 * will be gathered is calculated as: 0xffffffff - beg_value
757 */
758 writel(beg_value, dmc->base_drexi0 + DREX_PMCNT2_PPC);
759 writel(beg_value, dmc->base_drexi1 + DREX_PMCNT2_PPC);
760
761 /* Start all counters */
762 writel(PPC_ENABLE, dmc->base_drexi0 + DREX_PMNC_PPC);
763 writel(PPC_ENABLE, dmc->base_drexi1 + DREX_PMNC_PPC);
764}
765
766/**
767 * exynos5_dmc_perf_events_calc() - Calculate utilization
768 * @dmc: device for which the counters are going to be checked
769 * @diff_ts: time between last interrupt and current one
770 *
771 * Function which calculates needed utilization for the devfreq governor.
772 * It prepares values for 'busy_time' and 'total_time' based on elapsed time
773 * between interrupts, which approximates utilization.
774 */
775static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)
776{
777 /*
778 * This is a simple algorithm for managing traffic on DMC.
779 * When there is almost no load the counters overflow every 4s,
780 * no mater the DMC frequency.
781 * The high load might be approximated using linear function.
782 * Knowing that, simple calculation can provide 'busy_time' and
783 * 'total_time' to the devfreq governor which picks up target
784 * frequency.
785 * We want a fast ramp up and slow decay in frequency change function.
786 */
787 if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) {
788 /*
789 * Set higher utilization for the simple_ondemand governor.
790 * The governor should increase the frequency of the DMC.
791 */
792 dmc->load = 70;
793 dmc->total = 100;
794 } else {
795 /*
796 * Set low utilization for the simple_ondemand governor.
797 * The governor should decrease the frequency of the DMC.
798 */
799 dmc->load = 35;
800 dmc->total = 100;
801 }
802
803 dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);
804}
805
806/**
807 * exynos5_dmc_perf_events_check() - Checks the status of the counters
808 * @dmc: device for which the counters are going to be checked
809 *
810 * Function which is called from threaded IRQ to check the counters state
811 * and to call approximation for the needed utilization.
812 */
813static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)
814{
815 u32 val;
816 u64 diff_ts, ts;
817
818 ts = ktime_get_ns();
819
820 /* Stop all counters */
821 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
822 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
823
824 /* Check the source in interrupt flag registers (which channel) */
825 val = readl(dmc->base_drexi0 + DREX_FLAG_PPC);
826 if (val) {
827 diff_ts = ts - dmc->last_overflow_ts[0];
828 dmc->last_overflow_ts[0] = ts;
829 dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n", val);
830 } else {
831 val = readl(dmc->base_drexi1 + DREX_FLAG_PPC);
832 diff_ts = ts - dmc->last_overflow_ts[1];
833 dmc->last_overflow_ts[1] = ts;
834 dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n", val);
835 }
836
837 exynos5_dmc_perf_events_calc(dmc, diff_ts);
838
839 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
840}
841
842/**
843 * exynos5_dmc_enable_perf_events() - Enable performance events
844 * @dmc: device for which the counters are going to be checked
845 *
846 * Function which is setup needed environment and enables counters.
847 */
848static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)
849{
850 u64 ts;
851
852 /* Enable Performance Event Clock */
853 writel(PEREV_CLK_EN, dmc->base_drexi0 + DREX_PPCCLKCON);
854 writel(PEREV_CLK_EN, dmc->base_drexi1 + DREX_PPCCLKCON);
855
856 /* Select read transfers as performance event2 */
857 writel(READ_TRANSFER_CH0, dmc->base_drexi0 + DREX_PEREV2CONFIG);
858 writel(READ_TRANSFER_CH1, dmc->base_drexi1 + DREX_PEREV2CONFIG);
859
860 ts = ktime_get_ns();
861 dmc->last_overflow_ts[0] = ts;
862 dmc->last_overflow_ts[1] = ts;
863
864 /* Devfreq shouldn't be faster than initialization, play safe though. */
865 dmc->load = 99;
866 dmc->total = 100;
867}
868
869/**
870 * exynos5_dmc_disable_perf_events() - Disable performance events
871 * @dmc: device for which the counters are going to be checked
872 *
873 * Function which stops, disables performance event counters and interrupts.
874 */
875static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)
876{
877 /* Stop all counters */
878 writel(0, dmc->base_drexi0 + DREX_PMNC_PPC);
879 writel(0, dmc->base_drexi1 + DREX_PMNC_PPC);
880
881 /* Disable interrupts for counter 2 */
882 writel(PERF_CNT2, dmc->base_drexi0 + DREX_INTENC_PPC);
883 writel(PERF_CNT2, dmc->base_drexi1 + DREX_INTENC_PPC);
884
885 /* Disable counter 2 and CCNT */
886 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_CNTENC_PPC);
887 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_CNTENC_PPC);
888
889 /* Clear overflow flag for all counters */
890 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi0 + DREX_FLAG_PPC);
891 writel(PERF_CNT2 | PERF_CCNT, dmc->base_drexi1 + DREX_FLAG_PPC);
892}
893
894/**
895 * exynos5_dmc_get_status() - Read current DMC performance statistics.
896 * @dev: device for which the statistics are requested
897 * @stat: structure which has statistic fields
898 *
899 * Function reads the DMC performance counters and calculates 'busy_time'
900 * and 'total_time'. To protect from overflow, the values are shifted right
901 * by 10. After read out the counters are setup to count again.
902 */
903static int exynos5_dmc_get_status(struct device *dev,
904 struct devfreq_dev_status *stat)
905{
906 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
907 unsigned long load, total;
908 int ret;
909
910 if (dmc->in_irq_mode) {
911 stat->current_frequency = dmc->curr_rate;
912 stat->busy_time = dmc->load;
913 stat->total_time = dmc->total;
914 } else {
915 ret = exynos5_counters_get(dmc, &load, &total);
916 if (ret < 0)
917 return -EINVAL;
918
919 /* To protect from overflow, divide by 1024 */
920 stat->busy_time = load >> 10;
921 stat->total_time = total >> 10;
922
923 ret = exynos5_counters_set_event(dmc);
924 if (ret < 0) {
925 dev_err(dev, "could not set event counter\n");
926 return ret;
927 }
928 }
929
930 return 0;
931}
932
933/**
934 * exynos5_dmc_get_cur_freq() - Function returns current DMC frequency
935 * @dev: device for which the framework checks operating frequency
936 * @freq: returned frequency value
937 *
938 * It returns the currently used frequency of the DMC. The real operating
939 * frequency might be lower when the clock source value could not be divided
940 * to the requested value.
941 */
942static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq)
943{
944 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
945
946 mutex_lock(&dmc->lock);
947 *freq = dmc->curr_rate;
948 mutex_unlock(&dmc->lock);
949
950 return 0;
951}
952
953/**
954 * exynos5_dmc_df_profile - Devfreq governor's profile structure
955 *
956 * It provides to the devfreq framework needed functions and polling period.
957 */
958static struct devfreq_dev_profile exynos5_dmc_df_profile = {
959 .timer = DEVFREQ_TIMER_DELAYED,
960 .target = exynos5_dmc_target,
961 .get_dev_status = exynos5_dmc_get_status,
962 .get_cur_freq = exynos5_dmc_get_cur_freq,
963};
964
965/**
966 * exynos5_dmc_align_initial_frequency() - Align initial frequency value
967 * @dmc: device for which the frequency is going to be set
968 * @bootloader_init_freq: initial frequency set by the bootloader in KHz
969 *
970 * The initial bootloader frequency, which is present during boot, might be
971 * different that supported frequency values in the driver. It is possible
972 * due to different PLL settings or used PLL as a source.
973 * This function provides the 'initial_freq' for the devfreq framework
974 * statistics engine which supports only registered values. Thus, some alignment
975 * must be made.
976 */
977static unsigned long
978exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,
979 unsigned long bootloader_init_freq)
980{
981 unsigned long aligned_freq;
982 int idx;
983
984 idx = find_target_freq_idx(dmc, bootloader_init_freq);
985 if (idx >= 0)
986 aligned_freq = dmc->opp[idx].freq_hz;
987 else
988 aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;
989
990 return aligned_freq;
991}
992
993/**
994 * create_timings_aligned() - Create register values and align with standard
995 * @dmc: device for which the frequency is going to be set
996 * @idx: speed bin in the OPP table
997 * @clk_period_ps: the period of the clock, known as tCK
998 *
999 * The function calculates timings and creates a register value ready for
1000 * a frequency transition. The register contains a few timings. They are
1001 * shifted by a known offset. The timing value is calculated based on memory
1002 * specyfication: minimal time required and minimal cycles required.
1003 */
1004static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,
1005 u32 *reg_timing_data, u32 *reg_timing_power,
1006 u32 clk_period_ps)
1007{
1008 u32 val;
1009 const struct timing_reg *reg;
1010
1011 if (clk_period_ps == 0)
1012 return -EINVAL;
1013
1014 *reg_timing_row = 0;
1015 *reg_timing_data = 0;
1016 *reg_timing_power = 0;
1017
1018 val = dmc->timings->tRFC / clk_period_ps;
1019 val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;
1020 val = max(val, dmc->min_tck->tRFC);
1021 reg = &timing_row[0];
1022 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1023
1024 val = dmc->timings->tRRD / clk_period_ps;
1025 val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;
1026 val = max(val, dmc->min_tck->tRRD);
1027 reg = &timing_row[1];
1028 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1029
1030 val = dmc->timings->tRPab / clk_period_ps;
1031 val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;
1032 val = max(val, dmc->min_tck->tRPab);
1033 reg = &timing_row[2];
1034 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1035
1036 val = dmc->timings->tRCD / clk_period_ps;
1037 val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;
1038 val = max(val, dmc->min_tck->tRCD);
1039 reg = &timing_row[3];
1040 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1041
1042 val = dmc->timings->tRC / clk_period_ps;
1043 val += dmc->timings->tRC % clk_period_ps ? 1 : 0;
1044 val = max(val, dmc->min_tck->tRC);
1045 reg = &timing_row[4];
1046 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1047
1048 val = dmc->timings->tRAS / clk_period_ps;
1049 val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;
1050 val = max(val, dmc->min_tck->tRAS);
1051 reg = &timing_row[5];
1052 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1053
1054 /* data related timings */
1055 val = dmc->timings->tWTR / clk_period_ps;
1056 val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;
1057 val = max(val, dmc->min_tck->tWTR);
1058 reg = &timing_data[0];
1059 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1060
1061 val = dmc->timings->tWR / clk_period_ps;
1062 val += dmc->timings->tWR % clk_period_ps ? 1 : 0;
1063 val = max(val, dmc->min_tck->tWR);
1064 reg = &timing_data[1];
1065 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1066
1067 val = dmc->timings->tRTP / clk_period_ps;
1068 val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;
1069 val = max(val, dmc->min_tck->tRTP);
1070 reg = &timing_data[2];
1071 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1072
1073 val = dmc->timings->tW2W_C2C / clk_period_ps;
1074 val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;
1075 val = max(val, dmc->min_tck->tW2W_C2C);
1076 reg = &timing_data[3];
1077 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1078
1079 val = dmc->timings->tR2R_C2C / clk_period_ps;
1080 val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;
1081 val = max(val, dmc->min_tck->tR2R_C2C);
1082 reg = &timing_data[4];
1083 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1084
1085 val = dmc->timings->tWL / clk_period_ps;
1086 val += dmc->timings->tWL % clk_period_ps ? 1 : 0;
1087 val = max(val, dmc->min_tck->tWL);
1088 reg = &timing_data[5];
1089 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1090
1091 val = dmc->timings->tDQSCK / clk_period_ps;
1092 val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;
1093 val = max(val, dmc->min_tck->tDQSCK);
1094 reg = &timing_data[6];
1095 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1096
1097 val = dmc->timings->tRL / clk_period_ps;
1098 val += dmc->timings->tRL % clk_period_ps ? 1 : 0;
1099 val = max(val, dmc->min_tck->tRL);
1100 reg = &timing_data[7];
1101 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1102
1103 /* power related timings */
1104 val = dmc->timings->tFAW / clk_period_ps;
1105 val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;
1106 val = max(val, dmc->min_tck->tFAW);
1107 reg = &timing_power[0];
1108 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1109
1110 val = dmc->timings->tXSR / clk_period_ps;
1111 val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;
1112 val = max(val, dmc->min_tck->tXSR);
1113 reg = &timing_power[1];
1114 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1115
1116 val = dmc->timings->tXP / clk_period_ps;
1117 val += dmc->timings->tXP % clk_period_ps ? 1 : 0;
1118 val = max(val, dmc->min_tck->tXP);
1119 reg = &timing_power[2];
1120 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1121
1122 val = dmc->timings->tCKE / clk_period_ps;
1123 val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;
1124 val = max(val, dmc->min_tck->tCKE);
1125 reg = &timing_power[3];
1126 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1127
1128 val = dmc->timings->tMRD / clk_period_ps;
1129 val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;
1130 val = max(val, dmc->min_tck->tMRD);
1131 reg = &timing_power[4];
1132 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1133
1134 return 0;
1135}
1136
1137/**
1138 * of_get_dram_timings() - helper function for parsing DT settings for DRAM
1139 * @dmc: device for which the frequency is going to be set
1140 *
1141 * The function parses DT entries with DRAM information.
1142 */
1143static int of_get_dram_timings(struct exynos5_dmc *dmc)
1144{
1145 int ret = 0;
1146 int idx;
1147 struct device_node *np_ddr;
1148 u32 freq_mhz, clk_period_ps;
1149
1150 np_ddr = of_parse_phandle(dmc->dev->of_node, "device-handle", 0);
1151 if (!np_ddr) {
1152 dev_warn(dmc->dev, "could not find 'device-handle' in DT\n");
1153 return -EINVAL;
1154 }
1155
1156 dmc->timing_row = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1157 sizeof(u32), GFP_KERNEL);
1158 if (!dmc->timing_row)
1159 return -ENOMEM;
1160
1161 dmc->timing_data = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1162 sizeof(u32), GFP_KERNEL);
1163 if (!dmc->timing_data)
1164 return -ENOMEM;
1165
1166 dmc->timing_power = devm_kmalloc_array(dmc->dev, TIMING_COUNT,
1167 sizeof(u32), GFP_KERNEL);
1168 if (!dmc->timing_power)
1169 return -ENOMEM;
1170
1171 dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dmc->dev,
1172 DDR_TYPE_LPDDR3,
1173 &dmc->timings_arr_size);
1174 if (!dmc->timings) {
1175 of_node_put(np_ddr);
1176 dev_warn(dmc->dev, "could not get timings from DT\n");
1177 return -EINVAL;
1178 }
1179
1180 dmc->min_tck = of_lpddr3_get_min_tck(np_ddr, dmc->dev);
1181 if (!dmc->min_tck) {
1182 of_node_put(np_ddr);
1183 dev_warn(dmc->dev, "could not get tck from DT\n");
1184 return -EINVAL;
1185 }
1186
1187 /* Sorted array of OPPs with frequency ascending */
1188 for (idx = 0; idx < dmc->opp_count; idx++) {
1189 freq_mhz = dmc->opp[idx].freq_hz / 1000000;
1190 clk_period_ps = 1000000 / freq_mhz;
1191
1192 ret = create_timings_aligned(dmc, &dmc->timing_row[idx],
1193 &dmc->timing_data[idx],
1194 &dmc->timing_power[idx],
1195 clk_period_ps);
1196 }
1197
1198 of_node_put(np_ddr);
1199
1200 /* Take the highest frequency's timings as 'bypass' */
1201 dmc->bypass_timing_row = dmc->timing_row[idx - 1];
1202 dmc->bypass_timing_data = dmc->timing_data[idx - 1];
1203 dmc->bypass_timing_power = dmc->timing_power[idx - 1];
1204
1205 return ret;
1206}
1207
1208/**
1209 * exynos5_dmc_init_clks() - Initialize clocks needed for DMC operation.
1210 * @dmc: DMC structure containing needed fields
1211 *
1212 * Get the needed clocks defined in DT device, enable and set the right parents.
1213 * Read current frequency and initialize the initial rate for governor.
1214 */
1215static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)
1216{
1217 int ret;
1218 unsigned long target_volt = 0;
1219 unsigned long target_rate = 0;
1220 unsigned int tmp;
1221
1222 dmc->fout_spll = devm_clk_get(dmc->dev, "fout_spll");
1223 if (IS_ERR(dmc->fout_spll))
1224 return PTR_ERR(dmc->fout_spll);
1225
1226 dmc->fout_bpll = devm_clk_get(dmc->dev, "fout_bpll");
1227 if (IS_ERR(dmc->fout_bpll))
1228 return PTR_ERR(dmc->fout_bpll);
1229
1230 dmc->mout_mclk_cdrex = devm_clk_get(dmc->dev, "mout_mclk_cdrex");
1231 if (IS_ERR(dmc->mout_mclk_cdrex))
1232 return PTR_ERR(dmc->mout_mclk_cdrex);
1233
1234 dmc->mout_bpll = devm_clk_get(dmc->dev, "mout_bpll");
1235 if (IS_ERR(dmc->mout_bpll))
1236 return PTR_ERR(dmc->mout_bpll);
1237
1238 dmc->mout_mx_mspll_ccore = devm_clk_get(dmc->dev,
1239 "mout_mx_mspll_ccore");
1240 if (IS_ERR(dmc->mout_mx_mspll_ccore))
1241 return PTR_ERR(dmc->mout_mx_mspll_ccore);
1242
1243 dmc->mout_spll = devm_clk_get(dmc->dev, "ff_dout_spll2");
1244 if (IS_ERR(dmc->mout_spll)) {
1245 dmc->mout_spll = devm_clk_get(dmc->dev, "mout_sclk_spll");
1246 if (IS_ERR(dmc->mout_spll))
1247 return PTR_ERR(dmc->mout_spll);
1248 }
1249
1250 /*
1251 * Convert frequency to KHz values and set it for the governor.
1252 */
1253 dmc->curr_rate = clk_get_rate(dmc->mout_mclk_cdrex);
1254 dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, dmc->curr_rate);
1255 exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;
1256
1257 ret = exynos5_dmc_get_volt_freq(dmc, &dmc->curr_rate, &target_rate,
1258 &target_volt, 0);
1259 if (ret)
1260 return ret;
1261
1262 dmc->curr_volt = target_volt;
1263
1264 clk_set_parent(dmc->mout_mx_mspll_ccore, dmc->mout_spll);
1265
1266 dmc->bypass_rate = clk_get_rate(dmc->mout_mx_mspll_ccore);
1267
1268 clk_prepare_enable(dmc->fout_bpll);
1269 clk_prepare_enable(dmc->mout_bpll);
1270
1271 /*
1272 * Some bootloaders do not set clock routes correctly.
1273 * Stop one path in clocks to PHY.
1274 */
1275 regmap_read(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, &tmp);
1276 tmp &= ~(BIT(1) | BIT(0));
1277 regmap_write(dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, tmp);
1278
1279 return 0;
1280}
1281
1282/**
1283 * exynos5_performance_counters_init() - Initializes performance DMC's counters
1284 * @dmc: DMC for which it does the setup
1285 *
1286 * Initialization of performance counters in DMC for estimating usage.
1287 * The counter's values are used for calculation of a memory bandwidth and based
1288 * on that the governor changes the frequency.
1289 * The counters are not used when the governor is GOVERNOR_USERSPACE.
1290 */
1291static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)
1292{
1293 int counters_size;
1294 int ret, i;
1295
1296 dmc->num_counters = devfreq_event_get_edev_count(dmc->dev);
1297 if (dmc->num_counters < 0) {
1298 dev_err(dmc->dev, "could not get devfreq-event counters\n");
1299 return dmc->num_counters;
1300 }
1301
1302 counters_size = sizeof(struct devfreq_event_dev) * dmc->num_counters;
1303 dmc->counter = devm_kzalloc(dmc->dev, counters_size, GFP_KERNEL);
1304 if (!dmc->counter)
1305 return -ENOMEM;
1306
1307 for (i = 0; i < dmc->num_counters; i++) {
1308 dmc->counter[i] =
1309 devfreq_event_get_edev_by_phandle(dmc->dev, i);
1310 if (IS_ERR_OR_NULL(dmc->counter[i]))
1311 return -EPROBE_DEFER;
1312 }
1313
1314 ret = exynos5_counters_enable_edev(dmc);
1315 if (ret < 0) {
1316 dev_err(dmc->dev, "could not enable event counter\n");
1317 return ret;
1318 }
1319
1320 ret = exynos5_counters_set_event(dmc);
1321 if (ret < 0) {
1322 exynos5_counters_disable_edev(dmc);
1323 dev_err(dmc->dev, "could not set event counter\n");
1324 return ret;
1325 }
1326
1327 return 0;
1328}
1329
1330/**
1331 * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC
1332 * @dmc: device which is used for changing this feature
1333 * @set: a boolean state passing enable/disable request
1334 *
1335 * There is a need of pausing DREX DMC when divider or MUX in clock tree
1336 * changes its configuration. In such situation access to the memory is blocked
1337 * in DMC automatically. This feature is used when clock frequency change
1338 * request appears and touches clock tree.
1339 */
1340static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)
1341{
1342 unsigned int val;
1343 int ret;
1344
1345 ret = regmap_read(dmc->clk_regmap, CDREX_PAUSE, &val);
1346 if (ret)
1347 return ret;
1348
1349 val |= 1UL;
1350 regmap_write(dmc->clk_regmap, CDREX_PAUSE, val);
1351
1352 return 0;
1353}
1354
1355static irqreturn_t dmc_irq_thread(int irq, void *priv)
1356{
1357 int res;
1358 struct exynos5_dmc *dmc = priv;
1359
1360 mutex_lock(&dmc->df->lock);
1361 exynos5_dmc_perf_events_check(dmc);
1362 res = update_devfreq(dmc->df);
1363 mutex_unlock(&dmc->df->lock);
1364
1365 if (res)
1366 dev_warn(dmc->dev, "devfreq failed with %d\n", res);
1367
1368 return IRQ_HANDLED;
1369}
1370
1371/**
1372 * exynos5_dmc_probe() - Probe function for the DMC driver
1373 * @pdev: platform device for which the driver is going to be initialized
1374 *
1375 * Initialize basic components: clocks, regulators, performance counters, etc.
1376 * Read out product version and based on the information setup
1377 * internal structures for the controller (frequency and voltage) and for DRAM
1378 * memory parameters: timings for each operating frequency.
1379 * Register new devfreq device for controlling DVFS of the DMC.
1380 */
1381static int exynos5_dmc_probe(struct platform_device *pdev)
1382{
1383 int ret = 0;
1384 struct device *dev = &pdev->dev;
1385 struct device_node *np = dev->of_node;
1386 struct exynos5_dmc *dmc;
1387 int irq[2];
1388
1389 dmc = devm_kzalloc(dev, sizeof(*dmc), GFP_KERNEL);
1390 if (!dmc)
1391 return -ENOMEM;
1392
1393 mutex_init(&dmc->lock);
1394
1395 dmc->dev = dev;
1396 platform_set_drvdata(pdev, dmc);
1397
1398 dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, 0);
1399 if (IS_ERR(dmc->base_drexi0))
1400 return PTR_ERR(dmc->base_drexi0);
1401
1402 dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, 1);
1403 if (IS_ERR(dmc->base_drexi1))
1404 return PTR_ERR(dmc->base_drexi1);
1405
1406 dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,
1407 "samsung,syscon-clk");
1408 if (IS_ERR(dmc->clk_regmap))
1409 return PTR_ERR(dmc->clk_regmap);
1410
1411 ret = exynos5_init_freq_table(dmc, &exynos5_dmc_df_profile);
1412 if (ret) {
1413 dev_warn(dev, "couldn't initialize frequency settings\n");
1414 return ret;
1415 }
1416
1417 dmc->vdd_mif = devm_regulator_get(dev, "vdd");
1418 if (IS_ERR(dmc->vdd_mif)) {
1419 ret = PTR_ERR(dmc->vdd_mif);
1420 return ret;
1421 }
1422
1423 ret = exynos5_dmc_init_clks(dmc);
1424 if (ret)
1425 return ret;
1426
1427 ret = of_get_dram_timings(dmc);
1428 if (ret) {
1429 dev_warn(dev, "couldn't initialize timings settings\n");
1430 goto remove_clocks;
1431 }
1432
1433 ret = exynos5_dmc_set_pause_on_switching(dmc);
1434 if (ret) {
1435 dev_warn(dev, "couldn't get access to PAUSE register\n");
1436 goto remove_clocks;
1437 }
1438
1439 /* There is two modes in which the driver works: polling or IRQ */
1440 irq[0] = platform_get_irq_byname(pdev, "drex_0");
1441 irq[1] = platform_get_irq_byname(pdev, "drex_1");
1442 if (irq[0] > 0 && irq[1] > 0 && irqmode) {
1443 ret = devm_request_threaded_irq(dev, irq[0], NULL,
1444 dmc_irq_thread, IRQF_ONESHOT,
1445 dev_name(dev), dmc);
1446 if (ret) {
1447 dev_err(dev, "couldn't grab IRQ\n");
1448 goto remove_clocks;
1449 }
1450
1451 ret = devm_request_threaded_irq(dev, irq[1], NULL,
1452 dmc_irq_thread, IRQF_ONESHOT,
1453 dev_name(dev), dmc);
1454 if (ret) {
1455 dev_err(dev, "couldn't grab IRQ\n");
1456 goto remove_clocks;
1457 }
1458
1459 /*
1460 * Setup default thresholds for the devfreq governor.
1461 * The values are chosen based on experiments.
1462 */
1463 dmc->gov_data.upthreshold = 55;
1464 dmc->gov_data.downdifferential = 5;
1465
1466 exynos5_dmc_enable_perf_events(dmc);
1467
1468 dmc->in_irq_mode = 1;
1469 } else {
1470 ret = exynos5_performance_counters_init(dmc);
1471 if (ret) {
1472 dev_warn(dev, "couldn't probe performance counters\n");
1473 goto remove_clocks;
1474 }
1475
1476 /*
1477 * Setup default thresholds for the devfreq governor.
1478 * The values are chosen based on experiments.
1479 */
1480 dmc->gov_data.upthreshold = 10;
1481 dmc->gov_data.downdifferential = 5;
1482
1483 exynos5_dmc_df_profile.polling_ms = 100;
1484 }
1485
1486 dmc->df = devm_devfreq_add_device(dev, &exynos5_dmc_df_profile,
1487 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1488 &dmc->gov_data);
1489
1490 if (IS_ERR(dmc->df)) {
1491 ret = PTR_ERR(dmc->df);
1492 goto err_devfreq_add;
1493 }
1494
1495 if (dmc->in_irq_mode)
1496 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
1497
1498 dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode);
1499
1500 return 0;
1501
1502err_devfreq_add:
1503 if (dmc->in_irq_mode)
1504 exynos5_dmc_disable_perf_events(dmc);
1505 else
1506 exynos5_counters_disable_edev(dmc);
1507remove_clocks:
1508 clk_disable_unprepare(dmc->mout_bpll);
1509 clk_disable_unprepare(dmc->fout_bpll);
1510
1511 return ret;
1512}
1513
1514/**
1515 * exynos5_dmc_remove() - Remove function for the platform device
1516 * @pdev: platform device which is going to be removed
1517 *
1518 * The function relies on 'devm' framework function which automatically
1519 * clean the device's resources. It just calls explicitly disable function for
1520 * the performance counters.
1521 */
1522static int exynos5_dmc_remove(struct platform_device *pdev)
1523{
1524 struct exynos5_dmc *dmc = dev_get_drvdata(&pdev->dev);
1525
1526 if (dmc->in_irq_mode)
1527 exynos5_dmc_disable_perf_events(dmc);
1528 else
1529 exynos5_counters_disable_edev(dmc);
1530
1531 clk_disable_unprepare(dmc->mout_bpll);
1532 clk_disable_unprepare(dmc->fout_bpll);
1533
1534 dev_pm_opp_remove_table(dmc->dev);
1535
1536 return 0;
1537}
1538
1539static const struct of_device_id exynos5_dmc_of_match[] = {
1540 { .compatible = "samsung,exynos5422-dmc", },
1541 { },
1542};
1543MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match);
1544
1545static struct platform_driver exynos5_dmc_platdrv = {
1546 .probe = exynos5_dmc_probe,
1547 .remove = exynos5_dmc_remove,
1548 .driver = {
1549 .name = "exynos5-dmc",
1550 .of_match_table = exynos5_dmc_of_match,
1551 },
1552};
1553module_platform_driver(exynos5_dmc_platdrv);
1554MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change");
1555MODULE_LICENSE("GPL v2");
1556MODULE_AUTHOR("Lukasz Luba");