Loading...
Note: File does not exist in v3.1.
1/*
2 * Copyright (C) 2015-2016 Socionext Inc.
3 * Author: Masahiro Yamada <yamada.masahiro@socionext.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16#define pr_fmt(fmt) "uniphier: " fmt
17
18#include <linux/init.h>
19#include <linux/io.h>
20#include <linux/log2.h>
21#include <linux/of_address.h>
22#include <linux/slab.h>
23#include <asm/hardware/cache-uniphier.h>
24#include <asm/outercache.h>
25
26/* control registers */
27#define UNIPHIER_SSCC 0x0 /* Control Register */
28#define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */
29#define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */
30#define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */
31#define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */
32#define UNIPHIER_SSCC_ON BIT(0) /* enable cache */
33#define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */
34#define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */
35
36/* revision registers */
37#define UNIPHIER_SSCID 0x0 /* ID Register */
38
39/* operation registers */
40#define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */
41#define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */
42#define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */
43#define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */
44#define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */
45#define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */
46#define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */
47#define UNIPHIER_SSCOQM_S_MASK (0x3 << 17)
48#define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17)
49#define UNIPHIER_SSCOQM_S_ALL (0x1 << 17)
50#define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */
51#define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */
52#define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */
53#define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */
54#define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */
55#define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */
56#define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/
57#define UNIPHIER_SSCOPPQSEF_FE BIT(1)
58#define UNIPHIER_SSCOPPQSEF_OE BIT(0)
59#define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */
60#define UNIPHIER_SSCOLPQS_EF BIT(2)
61#define UNIPHIER_SSCOLPQS_EST BIT(1)
62#define UNIPHIER_SSCOLPQS_QST BIT(0)
63
64/* Is the operation region specified by address range? */
65#define UNIPHIER_SSCOQM_S_IS_RANGE(op) \
66 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE)
67
68/**
69 * uniphier_cache_data - UniPhier outer cache specific data
70 *
71 * @ctrl_base: virtual base address of control registers
72 * @rev_base: virtual base address of revision registers
73 * @op_base: virtual base address of operation registers
74 * @way_present_mask: each bit specifies if the way is present
75 * @way_locked_mask: each bit specifies if the way is locked
76 * @nsets: number of associativity sets
77 * @line_size: line size in bytes
78 * @range_op_max_size: max size that can be handled by a single range operation
79 * @list: list node to include this level in the whole cache hierarchy
80 */
81struct uniphier_cache_data {
82 void __iomem *ctrl_base;
83 void __iomem *rev_base;
84 void __iomem *op_base;
85 void __iomem *way_ctrl_base;
86 u32 way_present_mask;
87 u32 way_locked_mask;
88 u32 nsets;
89 u32 line_size;
90 u32 range_op_max_size;
91 struct list_head list;
92};
93
94/*
95 * List of the whole outer cache hierarchy. This list is only modified during
96 * the early boot stage, so no mutex is taken for the access to the list.
97 */
98static LIST_HEAD(uniphier_cache_list);
99
100/**
101 * __uniphier_cache_sync - perform a sync point for a particular cache level
102 *
103 * @data: cache controller specific data
104 */
105static void __uniphier_cache_sync(struct uniphier_cache_data *data)
106{
107 /* This sequence need not be atomic. Do not disable IRQ. */
108 writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC,
109 data->op_base + UNIPHIER_SSCOPE);
110 /* need a read back to confirm */
111 readl_relaxed(data->op_base + UNIPHIER_SSCOPE);
112}
113
114/**
115 * __uniphier_cache_maint_common - run a queue operation for a particular level
116 *
117 * @data: cache controller specific data
118 * @start: start address of range operation (don't care for "all" operation)
119 * @size: data size of range operation (don't care for "all" operation)
120 * @operation: flags to specify the desired cache operation
121 */
122static void __uniphier_cache_maint_common(struct uniphier_cache_data *data,
123 unsigned long start,
124 unsigned long size,
125 u32 operation)
126{
127 unsigned long flags;
128
129 /*
130 * No spin lock is necessary here because:
131 *
132 * [1] This outer cache controller is able to accept maintenance
133 * operations from multiple CPUs at a time in an SMP system; if a
134 * maintenance operation is under way and another operation is issued,
135 * the new one is stored in the queue. The controller performs one
136 * operation after another. If the queue is full, the status register,
137 * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has
138 * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have
139 * different instances for each CPU, i.e. each CPU can track the status
140 * of the maintenance operations triggered by itself.
141 *
142 * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ,
143 * SSCOQWN}, are shared between multiple CPUs, but the hardware still
144 * guarantees the registration sequence is atomic; the write access to
145 * them are arbitrated by the hardware. The first accessor to the
146 * register, UNIPHIER_SSCOQM, holds the access right and it is released
147 * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU
148 * is holding the access right, other CPUs fail to register operations.
149 * One CPU should not hold the access right for a long time, so local
150 * IRQs should be disabled while the following sequence.
151 */
152 local_irq_save(flags);
153
154 /* clear the complete notification flag */
155 writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS);
156
157 do {
158 /* set cache operation */
159 writel_relaxed(UNIPHIER_SSCOQM_CE | operation,
160 data->op_base + UNIPHIER_SSCOQM);
161
162 /* set address range if needed */
163 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) {
164 writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD);
165 writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ);
166 }
167 } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) &
168 (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE)));
169
170 /* wait until the operation is completed */
171 while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) !=
172 UNIPHIER_SSCOLPQS_EF))
173 cpu_relax();
174
175 local_irq_restore(flags);
176}
177
178static void __uniphier_cache_maint_all(struct uniphier_cache_data *data,
179 u32 operation)
180{
181 __uniphier_cache_maint_common(data, 0, 0,
182 UNIPHIER_SSCOQM_S_ALL | operation);
183
184 __uniphier_cache_sync(data);
185}
186
187static void __uniphier_cache_maint_range(struct uniphier_cache_data *data,
188 unsigned long start, unsigned long end,
189 u32 operation)
190{
191 unsigned long size;
192
193 /*
194 * If the start address is not aligned,
195 * perform a cache operation for the first cache-line
196 */
197 start = start & ~(data->line_size - 1);
198
199 size = end - start;
200
201 if (unlikely(size >= (unsigned long)(-data->line_size))) {
202 /* this means cache operation for all range */
203 __uniphier_cache_maint_all(data, operation);
204 return;
205 }
206
207 /*
208 * If the end address is not aligned,
209 * perform a cache operation for the last cache-line
210 */
211 size = ALIGN(size, data->line_size);
212
213 while (size) {
214 unsigned long chunk_size = min_t(unsigned long, size,
215 data->range_op_max_size);
216
217 __uniphier_cache_maint_common(data, start, chunk_size,
218 UNIPHIER_SSCOQM_S_RANGE | operation);
219
220 start += chunk_size;
221 size -= chunk_size;
222 }
223
224 __uniphier_cache_sync(data);
225}
226
227static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on)
228{
229 u32 val = 0;
230
231 if (on)
232 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON;
233
234 writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC);
235}
236
237static void __init __uniphier_cache_set_locked_ways(
238 struct uniphier_cache_data *data,
239 u32 way_mask)
240{
241 unsigned int cpu;
242
243 data->way_locked_mask = way_mask & data->way_present_mask;
244
245 for_each_possible_cpu(cpu)
246 writel_relaxed(~data->way_locked_mask & data->way_present_mask,
247 data->way_ctrl_base + 4 * cpu);
248}
249
250static void uniphier_cache_maint_range(unsigned long start, unsigned long end,
251 u32 operation)
252{
253 struct uniphier_cache_data *data;
254
255 list_for_each_entry(data, &uniphier_cache_list, list)
256 __uniphier_cache_maint_range(data, start, end, operation);
257}
258
259static void uniphier_cache_maint_all(u32 operation)
260{
261 struct uniphier_cache_data *data;
262
263 list_for_each_entry(data, &uniphier_cache_list, list)
264 __uniphier_cache_maint_all(data, operation);
265}
266
267static void uniphier_cache_inv_range(unsigned long start, unsigned long end)
268{
269 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV);
270}
271
272static void uniphier_cache_clean_range(unsigned long start, unsigned long end)
273{
274 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN);
275}
276
277static void uniphier_cache_flush_range(unsigned long start, unsigned long end)
278{
279 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH);
280}
281
282static void __init uniphier_cache_inv_all(void)
283{
284 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV);
285}
286
287static void uniphier_cache_flush_all(void)
288{
289 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH);
290}
291
292static void uniphier_cache_disable(void)
293{
294 struct uniphier_cache_data *data;
295
296 list_for_each_entry_reverse(data, &uniphier_cache_list, list)
297 __uniphier_cache_enable(data, false);
298
299 uniphier_cache_flush_all();
300}
301
302static void __init uniphier_cache_enable(void)
303{
304 struct uniphier_cache_data *data;
305
306 uniphier_cache_inv_all();
307
308 list_for_each_entry(data, &uniphier_cache_list, list) {
309 __uniphier_cache_enable(data, true);
310 __uniphier_cache_set_locked_ways(data, 0);
311 }
312}
313
314static void uniphier_cache_sync(void)
315{
316 struct uniphier_cache_data *data;
317
318 list_for_each_entry(data, &uniphier_cache_list, list)
319 __uniphier_cache_sync(data);
320}
321
322static const struct of_device_id uniphier_cache_match[] __initconst = {
323 { .compatible = "socionext,uniphier-system-cache" },
324 { /* sentinel */ }
325};
326
327static int __init __uniphier_cache_init(struct device_node *np,
328 unsigned int *cache_level)
329{
330 struct uniphier_cache_data *data;
331 u32 level, cache_size;
332 struct device_node *next_np;
333 int ret = 0;
334
335 if (!of_match_node(uniphier_cache_match, np)) {
336 pr_err("L%d: not compatible with uniphier cache\n",
337 *cache_level);
338 return -EINVAL;
339 }
340
341 if (of_property_read_u32(np, "cache-level", &level)) {
342 pr_err("L%d: cache-level is not specified\n", *cache_level);
343 return -EINVAL;
344 }
345
346 if (level != *cache_level) {
347 pr_err("L%d: cache-level is unexpected value %d\n",
348 *cache_level, level);
349 return -EINVAL;
350 }
351
352 if (!of_property_read_bool(np, "cache-unified")) {
353 pr_err("L%d: cache-unified is not specified\n", *cache_level);
354 return -EINVAL;
355 }
356
357 data = kzalloc(sizeof(*data), GFP_KERNEL);
358 if (!data)
359 return -ENOMEM;
360
361 if (of_property_read_u32(np, "cache-line-size", &data->line_size) ||
362 !is_power_of_2(data->line_size)) {
363 pr_err("L%d: cache-line-size is unspecified or invalid\n",
364 *cache_level);
365 ret = -EINVAL;
366 goto err;
367 }
368
369 if (of_property_read_u32(np, "cache-sets", &data->nsets) ||
370 !is_power_of_2(data->nsets)) {
371 pr_err("L%d: cache-sets is unspecified or invalid\n",
372 *cache_level);
373 ret = -EINVAL;
374 goto err;
375 }
376
377 if (of_property_read_u32(np, "cache-size", &cache_size) ||
378 cache_size == 0 || cache_size % (data->nsets * data->line_size)) {
379 pr_err("L%d: cache-size is unspecified or invalid\n",
380 *cache_level);
381 ret = -EINVAL;
382 goto err;
383 }
384
385 data->way_present_mask =
386 ((u32)1 << cache_size / data->nsets / data->line_size) - 1;
387
388 data->ctrl_base = of_iomap(np, 0);
389 if (!data->ctrl_base) {
390 pr_err("L%d: failed to map control register\n", *cache_level);
391 ret = -ENOMEM;
392 goto err;
393 }
394
395 data->rev_base = of_iomap(np, 1);
396 if (!data->rev_base) {
397 pr_err("L%d: failed to map revision register\n", *cache_level);
398 ret = -ENOMEM;
399 goto err;
400 }
401
402 data->op_base = of_iomap(np, 2);
403 if (!data->op_base) {
404 pr_err("L%d: failed to map operation register\n", *cache_level);
405 ret = -ENOMEM;
406 goto err;
407 }
408
409 data->way_ctrl_base = data->ctrl_base + 0xc00;
410
411 if (*cache_level == 2) {
412 u32 revision = readl(data->rev_base + UNIPHIER_SSCID);
413 /*
414 * The size of range operation is limited to (1 << 22) or less
415 * for PH-sLD8 or older SoCs.
416 */
417 if (revision <= 0x16)
418 data->range_op_max_size = (u32)1 << 22;
419
420 /*
421 * Unfortunatly, the offset address of active way control base
422 * varies from SoC to SoC.
423 */
424 switch (revision) {
425 case 0x11: /* sLD3 */
426 data->way_ctrl_base = data->ctrl_base + 0x870;
427 break;
428 case 0x12: /* LD4 */
429 case 0x16: /* sld8 */
430 data->way_ctrl_base = data->ctrl_base + 0x840;
431 break;
432 default:
433 break;
434 }
435 }
436
437 data->range_op_max_size -= data->line_size;
438
439 INIT_LIST_HEAD(&data->list);
440 list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */
441
442 /*
443 * OK, this level has been successfully initialized. Look for the next
444 * level cache. Do not roll back even if the initialization of the
445 * next level cache fails because we want to continue with available
446 * cache levels.
447 */
448 next_np = of_find_next_cache_node(np);
449 if (next_np) {
450 (*cache_level)++;
451 ret = __uniphier_cache_init(next_np, cache_level);
452 }
453 of_node_put(next_np);
454
455 return ret;
456err:
457 iounmap(data->op_base);
458 iounmap(data->rev_base);
459 iounmap(data->ctrl_base);
460 kfree(data);
461
462 return ret;
463}
464
465int __init uniphier_cache_init(void)
466{
467 struct device_node *np = NULL;
468 unsigned int cache_level;
469 int ret = 0;
470
471 /* look for level 2 cache */
472 while ((np = of_find_matching_node(np, uniphier_cache_match)))
473 if (!of_property_read_u32(np, "cache-level", &cache_level) &&
474 cache_level == 2)
475 break;
476
477 if (!np)
478 return -ENODEV;
479
480 ret = __uniphier_cache_init(np, &cache_level);
481 of_node_put(np);
482
483 if (ret) {
484 /*
485 * Error out iif L2 initialization fails. Continue with any
486 * error on L3 or outer because they are optional.
487 */
488 if (cache_level == 2) {
489 pr_err("failed to initialize L2 cache\n");
490 return ret;
491 }
492
493 cache_level--;
494 ret = 0;
495 }
496
497 outer_cache.inv_range = uniphier_cache_inv_range;
498 outer_cache.clean_range = uniphier_cache_clean_range;
499 outer_cache.flush_range = uniphier_cache_flush_range;
500 outer_cache.flush_all = uniphier_cache_flush_all;
501 outer_cache.disable = uniphier_cache_disable;
502 outer_cache.sync = uniphier_cache_sync;
503
504 uniphier_cache_enable();
505
506 pr_info("enabled outer cache (cache level: %d)\n", cache_level);
507
508 return ret;
509}