Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2015-2016 Socionext Inc.
4 * Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5 */
6
7#define pr_fmt(fmt) "uniphier: " fmt
8
9#include <linux/bitops.h>
10#include <linux/init.h>
11#include <linux/io.h>
12#include <linux/log2.h>
13#include <linux/of_address.h>
14#include <linux/slab.h>
15#include <asm/hardware/cache-uniphier.h>
16#include <asm/outercache.h>
17
18/* control registers */
19#define UNIPHIER_SSCC 0x0 /* Control Register */
20#define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */
21#define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */
22#define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */
23#define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */
24#define UNIPHIER_SSCC_ON BIT(0) /* enable cache */
25#define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */
26#define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */
27
28/* revision registers */
29#define UNIPHIER_SSCID 0x0 /* ID Register */
30
31/* operation registers */
32#define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */
33#define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */
34#define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */
35#define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */
36#define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */
37#define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */
38#define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */
39#define UNIPHIER_SSCOQM_S_MASK (0x3 << 17)
40#define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17)
41#define UNIPHIER_SSCOQM_S_ALL (0x1 << 17)
42#define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */
43#define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */
44#define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */
45#define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */
46#define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */
47#define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */
48#define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/
49#define UNIPHIER_SSCOPPQSEF_FE BIT(1)
50#define UNIPHIER_SSCOPPQSEF_OE BIT(0)
51#define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */
52#define UNIPHIER_SSCOLPQS_EF BIT(2)
53#define UNIPHIER_SSCOLPQS_EST BIT(1)
54#define UNIPHIER_SSCOLPQS_QST BIT(0)
55
56/* Is the operation region specified by address range? */
57#define UNIPHIER_SSCOQM_S_IS_RANGE(op) \
58 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE)
59
60/**
61 * struct uniphier_cache_data - UniPhier outer cache specific data
62 *
63 * @ctrl_base: virtual base address of control registers
64 * @rev_base: virtual base address of revision registers
65 * @op_base: virtual base address of operation registers
66 * @way_ctrl_base: virtual address of the way control registers for this
67 * SoC revision
68 * @way_mask: each bit specifies if the way is present
69 * @nsets: number of associativity sets
70 * @line_size: line size in bytes
71 * @range_op_max_size: max size that can be handled by a single range operation
72 * @list: list node to include this level in the whole cache hierarchy
73 */
74struct uniphier_cache_data {
75 void __iomem *ctrl_base;
76 void __iomem *rev_base;
77 void __iomem *op_base;
78 void __iomem *way_ctrl_base;
79 u32 way_mask;
80 u32 nsets;
81 u32 line_size;
82 u32 range_op_max_size;
83 struct list_head list;
84};
85
86/*
87 * List of the whole outer cache hierarchy. This list is only modified during
88 * the early boot stage, so no mutex is taken for the access to the list.
89 */
90static LIST_HEAD(uniphier_cache_list);
91
92/**
93 * __uniphier_cache_sync - perform a sync point for a particular cache level
94 *
95 * @data: cache controller specific data
96 */
97static void __uniphier_cache_sync(struct uniphier_cache_data *data)
98{
99 /* This sequence need not be atomic. Do not disable IRQ. */
100 writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC,
101 data->op_base + UNIPHIER_SSCOPE);
102 /* need a read back to confirm */
103 readl_relaxed(data->op_base + UNIPHIER_SSCOPE);
104}
105
106/**
107 * __uniphier_cache_maint_common - run a queue operation for a particular level
108 *
109 * @data: cache controller specific data
110 * @start: start address of range operation (don't care for "all" operation)
111 * @size: data size of range operation (don't care for "all" operation)
112 * @operation: flags to specify the desired cache operation
113 */
114static void __uniphier_cache_maint_common(struct uniphier_cache_data *data,
115 unsigned long start,
116 unsigned long size,
117 u32 operation)
118{
119 unsigned long flags;
120
121 /*
122 * No spin lock is necessary here because:
123 *
124 * [1] This outer cache controller is able to accept maintenance
125 * operations from multiple CPUs at a time in an SMP system; if a
126 * maintenance operation is under way and another operation is issued,
127 * the new one is stored in the queue. The controller performs one
128 * operation after another. If the queue is full, the status register,
129 * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has
130 * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have
131 * different instances for each CPU, i.e. each CPU can track the status
132 * of the maintenance operations triggered by itself.
133 *
134 * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ,
135 * SSCOQWN}, are shared between multiple CPUs, but the hardware still
136 * guarantees the registration sequence is atomic; the write access to
137 * them are arbitrated by the hardware. The first accessor to the
138 * register, UNIPHIER_SSCOQM, holds the access right and it is released
139 * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU
140 * is holding the access right, other CPUs fail to register operations.
141 * One CPU should not hold the access right for a long time, so local
142 * IRQs should be disabled while the following sequence.
143 */
144 local_irq_save(flags);
145
146 /* clear the complete notification flag */
147 writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS);
148
149 do {
150 /* set cache operation */
151 writel_relaxed(UNIPHIER_SSCOQM_CE | operation,
152 data->op_base + UNIPHIER_SSCOQM);
153
154 /* set address range if needed */
155 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) {
156 writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD);
157 writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ);
158 }
159 } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) &
160 (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE)));
161
162 /* wait until the operation is completed */
163 while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) !=
164 UNIPHIER_SSCOLPQS_EF))
165 cpu_relax();
166
167 local_irq_restore(flags);
168}
169
170static void __uniphier_cache_maint_all(struct uniphier_cache_data *data,
171 u32 operation)
172{
173 __uniphier_cache_maint_common(data, 0, 0,
174 UNIPHIER_SSCOQM_S_ALL | operation);
175
176 __uniphier_cache_sync(data);
177}
178
179static void __uniphier_cache_maint_range(struct uniphier_cache_data *data,
180 unsigned long start, unsigned long end,
181 u32 operation)
182{
183 unsigned long size;
184
185 /*
186 * If the start address is not aligned,
187 * perform a cache operation for the first cache-line
188 */
189 start = start & ~(data->line_size - 1);
190
191 size = end - start;
192
193 if (unlikely(size >= (unsigned long)(-data->line_size))) {
194 /* this means cache operation for all range */
195 __uniphier_cache_maint_all(data, operation);
196 return;
197 }
198
199 /*
200 * If the end address is not aligned,
201 * perform a cache operation for the last cache-line
202 */
203 size = ALIGN(size, data->line_size);
204
205 while (size) {
206 unsigned long chunk_size = min_t(unsigned long, size,
207 data->range_op_max_size);
208
209 __uniphier_cache_maint_common(data, start, chunk_size,
210 UNIPHIER_SSCOQM_S_RANGE | operation);
211
212 start += chunk_size;
213 size -= chunk_size;
214 }
215
216 __uniphier_cache_sync(data);
217}
218
219static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on)
220{
221 u32 val = 0;
222
223 if (on)
224 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON;
225
226 writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC);
227}
228
229static void __init __uniphier_cache_set_active_ways(
230 struct uniphier_cache_data *data)
231{
232 unsigned int cpu;
233
234 for_each_possible_cpu(cpu)
235 writel_relaxed(data->way_mask, data->way_ctrl_base + 4 * cpu);
236}
237
238static void uniphier_cache_maint_range(unsigned long start, unsigned long end,
239 u32 operation)
240{
241 struct uniphier_cache_data *data;
242
243 list_for_each_entry(data, &uniphier_cache_list, list)
244 __uniphier_cache_maint_range(data, start, end, operation);
245}
246
247static void uniphier_cache_maint_all(u32 operation)
248{
249 struct uniphier_cache_data *data;
250
251 list_for_each_entry(data, &uniphier_cache_list, list)
252 __uniphier_cache_maint_all(data, operation);
253}
254
255static void uniphier_cache_inv_range(unsigned long start, unsigned long end)
256{
257 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV);
258}
259
260static void uniphier_cache_clean_range(unsigned long start, unsigned long end)
261{
262 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN);
263}
264
265static void uniphier_cache_flush_range(unsigned long start, unsigned long end)
266{
267 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH);
268}
269
270static void __init uniphier_cache_inv_all(void)
271{
272 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV);
273}
274
275static void uniphier_cache_flush_all(void)
276{
277 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH);
278}
279
280static void uniphier_cache_disable(void)
281{
282 struct uniphier_cache_data *data;
283
284 list_for_each_entry_reverse(data, &uniphier_cache_list, list)
285 __uniphier_cache_enable(data, false);
286
287 uniphier_cache_flush_all();
288}
289
290static void __init uniphier_cache_enable(void)
291{
292 struct uniphier_cache_data *data;
293
294 uniphier_cache_inv_all();
295
296 list_for_each_entry(data, &uniphier_cache_list, list) {
297 __uniphier_cache_enable(data, true);
298 __uniphier_cache_set_active_ways(data);
299 }
300}
301
302static void uniphier_cache_sync(void)
303{
304 struct uniphier_cache_data *data;
305
306 list_for_each_entry(data, &uniphier_cache_list, list)
307 __uniphier_cache_sync(data);
308}
309
310static const struct of_device_id uniphier_cache_match[] __initconst = {
311 { .compatible = "socionext,uniphier-system-cache" },
312 { /* sentinel */ }
313};
314
315static int __init __uniphier_cache_init(struct device_node *np,
316 unsigned int *cache_level)
317{
318 struct uniphier_cache_data *data;
319 u32 level, cache_size;
320 struct device_node *next_np;
321 int ret = 0;
322
323 if (!of_match_node(uniphier_cache_match, np)) {
324 pr_err("L%d: not compatible with uniphier cache\n",
325 *cache_level);
326 return -EINVAL;
327 }
328
329 if (of_property_read_u32(np, "cache-level", &level)) {
330 pr_err("L%d: cache-level is not specified\n", *cache_level);
331 return -EINVAL;
332 }
333
334 if (level != *cache_level) {
335 pr_err("L%d: cache-level is unexpected value %d\n",
336 *cache_level, level);
337 return -EINVAL;
338 }
339
340 if (!of_property_read_bool(np, "cache-unified")) {
341 pr_err("L%d: cache-unified is not specified\n", *cache_level);
342 return -EINVAL;
343 }
344
345 data = kzalloc(sizeof(*data), GFP_KERNEL);
346 if (!data)
347 return -ENOMEM;
348
349 if (of_property_read_u32(np, "cache-line-size", &data->line_size) ||
350 !is_power_of_2(data->line_size)) {
351 pr_err("L%d: cache-line-size is unspecified or invalid\n",
352 *cache_level);
353 ret = -EINVAL;
354 goto err;
355 }
356
357 if (of_property_read_u32(np, "cache-sets", &data->nsets) ||
358 !is_power_of_2(data->nsets)) {
359 pr_err("L%d: cache-sets is unspecified or invalid\n",
360 *cache_level);
361 ret = -EINVAL;
362 goto err;
363 }
364
365 if (of_property_read_u32(np, "cache-size", &cache_size) ||
366 cache_size == 0 || cache_size % (data->nsets * data->line_size)) {
367 pr_err("L%d: cache-size is unspecified or invalid\n",
368 *cache_level);
369 ret = -EINVAL;
370 goto err;
371 }
372
373 data->way_mask = GENMASK(cache_size / data->nsets / data->line_size - 1,
374 0);
375
376 data->ctrl_base = of_iomap(np, 0);
377 if (!data->ctrl_base) {
378 pr_err("L%d: failed to map control register\n", *cache_level);
379 ret = -ENOMEM;
380 goto err;
381 }
382
383 data->rev_base = of_iomap(np, 1);
384 if (!data->rev_base) {
385 pr_err("L%d: failed to map revision register\n", *cache_level);
386 ret = -ENOMEM;
387 goto err;
388 }
389
390 data->op_base = of_iomap(np, 2);
391 if (!data->op_base) {
392 pr_err("L%d: failed to map operation register\n", *cache_level);
393 ret = -ENOMEM;
394 goto err;
395 }
396
397 data->way_ctrl_base = data->ctrl_base + 0xc00;
398
399 if (*cache_level == 2) {
400 u32 revision = readl(data->rev_base + UNIPHIER_SSCID);
401 /*
402 * The size of range operation is limited to (1 << 22) or less
403 * for PH-sLD8 or older SoCs.
404 */
405 if (revision <= 0x16)
406 data->range_op_max_size = (u32)1 << 22;
407
408 /*
409 * Unfortunatly, the offset address of active way control base
410 * varies from SoC to SoC.
411 */
412 switch (revision) {
413 case 0x11: /* sLD3 */
414 data->way_ctrl_base = data->ctrl_base + 0x870;
415 break;
416 case 0x12: /* LD4 */
417 case 0x16: /* sld8 */
418 data->way_ctrl_base = data->ctrl_base + 0x840;
419 break;
420 default:
421 break;
422 }
423 }
424
425 data->range_op_max_size -= data->line_size;
426
427 INIT_LIST_HEAD(&data->list);
428 list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */
429
430 /*
431 * OK, this level has been successfully initialized. Look for the next
432 * level cache. Do not roll back even if the initialization of the
433 * next level cache fails because we want to continue with available
434 * cache levels.
435 */
436 next_np = of_find_next_cache_node(np);
437 if (next_np) {
438 (*cache_level)++;
439 ret = __uniphier_cache_init(next_np, cache_level);
440 }
441 of_node_put(next_np);
442
443 return ret;
444err:
445 iounmap(data->op_base);
446 iounmap(data->rev_base);
447 iounmap(data->ctrl_base);
448 kfree(data);
449
450 return ret;
451}
452
453int __init uniphier_cache_init(void)
454{
455 struct device_node *np = NULL;
456 unsigned int cache_level;
457 int ret = 0;
458
459 /* look for level 2 cache */
460 while ((np = of_find_matching_node(np, uniphier_cache_match)))
461 if (!of_property_read_u32(np, "cache-level", &cache_level) &&
462 cache_level == 2)
463 break;
464
465 if (!np)
466 return -ENODEV;
467
468 ret = __uniphier_cache_init(np, &cache_level);
469 of_node_put(np);
470
471 if (ret) {
472 /*
473 * Error out iif L2 initialization fails. Continue with any
474 * error on L3 or outer because they are optional.
475 */
476 if (cache_level == 2) {
477 pr_err("failed to initialize L2 cache\n");
478 return ret;
479 }
480
481 cache_level--;
482 ret = 0;
483 }
484
485 outer_cache.inv_range = uniphier_cache_inv_range;
486 outer_cache.clean_range = uniphier_cache_clean_range;
487 outer_cache.flush_range = uniphier_cache_flush_range;
488 outer_cache.flush_all = uniphier_cache_flush_all;
489 outer_cache.disable = uniphier_cache_disable;
490 outer_cache.sync = uniphier_cache_sync;
491
492 uniphier_cache_enable();
493
494 pr_info("enabled outer cache (cache level: %d)\n", cache_level);
495
496 return ret;
497}
1/*
2 * Copyright (C) 2015 Masahiro Yamada <yamada.masahiro@socionext.com>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 */
14
15#define pr_fmt(fmt) "uniphier: " fmt
16
17#include <linux/init.h>
18#include <linux/io.h>
19#include <linux/log2.h>
20#include <linux/of_address.h>
21#include <linux/slab.h>
22#include <asm/hardware/cache-uniphier.h>
23#include <asm/outercache.h>
24
25/* control registers */
26#define UNIPHIER_SSCC 0x0 /* Control Register */
27#define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */
28#define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */
29#define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */
30#define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */
31#define UNIPHIER_SSCC_ON BIT(0) /* enable cache */
32#define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */
33#define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */
34
35/* revision registers */
36#define UNIPHIER_SSCID 0x0 /* ID Register */
37
38/* operation registers */
39#define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */
40#define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */
41#define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */
42#define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */
43#define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */
44#define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */
45#define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */
46#define UNIPHIER_SSCOQM_TID_MASK (0x3 << 21)
47#define UNIPHIER_SSCOQM_TID_LRU_DATA (0x0 << 21)
48#define UNIPHIER_SSCOQM_TID_LRU_INST (0x1 << 21)
49#define UNIPHIER_SSCOQM_TID_WAY (0x2 << 21)
50#define UNIPHIER_SSCOQM_S_MASK (0x3 << 17)
51#define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17)
52#define UNIPHIER_SSCOQM_S_ALL (0x1 << 17)
53#define UNIPHIER_SSCOQM_S_WAY (0x2 << 17)
54#define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */
55#define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */
56#define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */
57#define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */
58#define UNIPHIER_SSCOQM_CM_PREFETCH 0x3 /* prefetch to cache */
59#define UNIPHIER_SSCOQM_CM_PREFETCH_BUF 0x4 /* prefetch to pf-buf */
60#define UNIPHIER_SSCOQM_CM_TOUCH 0x5 /* touch */
61#define UNIPHIER_SSCOQM_CM_TOUCH_ZERO 0x6 /* touch to zero */
62#define UNIPHIER_SSCOQM_CM_TOUCH_DIRTY 0x7 /* touch with dirty */
63#define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */
64#define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */
65#define UNIPHIER_SSCOQMASK 0x254 /* Cache Operation Queue Address Mask */
66#define UNIPHIER_SSCOQWN 0x258 /* Cache Operation Queue Way Number */
67#define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/
68#define UNIPHIER_SSCOPPQSEF_FE BIT(1)
69#define UNIPHIER_SSCOPPQSEF_OE BIT(0)
70#define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */
71#define UNIPHIER_SSCOLPQS_EF BIT(2)
72#define UNIPHIER_SSCOLPQS_EST BIT(1)
73#define UNIPHIER_SSCOLPQS_QST BIT(0)
74
75/* Is the touch/pre-fetch destination specified by ways? */
76#define UNIPHIER_SSCOQM_TID_IS_WAY(op) \
77 ((op & UNIPHIER_SSCOQM_TID_MASK) == UNIPHIER_SSCOQM_TID_WAY)
78/* Is the operation region specified by address range? */
79#define UNIPHIER_SSCOQM_S_IS_RANGE(op) \
80 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE)
81
82/**
83 * uniphier_cache_data - UniPhier outer cache specific data
84 *
85 * @ctrl_base: virtual base address of control registers
86 * @rev_base: virtual base address of revision registers
87 * @op_base: virtual base address of operation registers
88 * @way_present_mask: each bit specifies if the way is present
89 * @way_locked_mask: each bit specifies if the way is locked
90 * @nsets: number of associativity sets
91 * @line_size: line size in bytes
92 * @range_op_max_size: max size that can be handled by a single range operation
93 * @list: list node to include this level in the whole cache hierarchy
94 */
95struct uniphier_cache_data {
96 void __iomem *ctrl_base;
97 void __iomem *rev_base;
98 void __iomem *op_base;
99 u32 way_present_mask;
100 u32 way_locked_mask;
101 u32 nsets;
102 u32 line_size;
103 u32 range_op_max_size;
104 struct list_head list;
105};
106
107/*
108 * List of the whole outer cache hierarchy. This list is only modified during
109 * the early boot stage, so no mutex is taken for the access to the list.
110 */
111static LIST_HEAD(uniphier_cache_list);
112
113/**
114 * __uniphier_cache_sync - perform a sync point for a particular cache level
115 *
116 * @data: cache controller specific data
117 */
118static void __uniphier_cache_sync(struct uniphier_cache_data *data)
119{
120 /* This sequence need not be atomic. Do not disable IRQ. */
121 writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC,
122 data->op_base + UNIPHIER_SSCOPE);
123 /* need a read back to confirm */
124 readl_relaxed(data->op_base + UNIPHIER_SSCOPE);
125}
126
127/**
128 * __uniphier_cache_maint_common - run a queue operation for a particular level
129 *
130 * @data: cache controller specific data
131 * @start: start address of range operation (don't care for "all" operation)
132 * @size: data size of range operation (don't care for "all" operation)
133 * @operation: flags to specify the desired cache operation
134 */
135static void __uniphier_cache_maint_common(struct uniphier_cache_data *data,
136 unsigned long start,
137 unsigned long size,
138 u32 operation)
139{
140 unsigned long flags;
141
142 /*
143 * No spin lock is necessary here because:
144 *
145 * [1] This outer cache controller is able to accept maintenance
146 * operations from multiple CPUs at a time in an SMP system; if a
147 * maintenance operation is under way and another operation is issued,
148 * the new one is stored in the queue. The controller performs one
149 * operation after another. If the queue is full, the status register,
150 * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has
151 * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have
152 * different instances for each CPU, i.e. each CPU can track the status
153 * of the maintenance operations triggered by itself.
154 *
155 * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ,
156 * SSCOQWN}, are shared between multiple CPUs, but the hardware still
157 * guarantees the registration sequence is atomic; the write access to
158 * them are arbitrated by the hardware. The first accessor to the
159 * register, UNIPHIER_SSCOQM, holds the access right and it is released
160 * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU
161 * is holding the access right, other CPUs fail to register operations.
162 * One CPU should not hold the access right for a long time, so local
163 * IRQs should be disabled while the following sequence.
164 */
165 local_irq_save(flags);
166
167 /* clear the complete notification flag */
168 writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS);
169
170 do {
171 /* set cache operation */
172 writel_relaxed(UNIPHIER_SSCOQM_CE | operation,
173 data->op_base + UNIPHIER_SSCOQM);
174
175 /* set address range if needed */
176 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) {
177 writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD);
178 writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ);
179 }
180
181 /* set target ways if needed */
182 if (unlikely(UNIPHIER_SSCOQM_TID_IS_WAY(operation)))
183 writel_relaxed(data->way_locked_mask,
184 data->op_base + UNIPHIER_SSCOQWN);
185 } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) &
186 (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE)));
187
188 /* wait until the operation is completed */
189 while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) !=
190 UNIPHIER_SSCOLPQS_EF))
191 cpu_relax();
192
193 local_irq_restore(flags);
194}
195
196static void __uniphier_cache_maint_all(struct uniphier_cache_data *data,
197 u32 operation)
198{
199 __uniphier_cache_maint_common(data, 0, 0,
200 UNIPHIER_SSCOQM_S_ALL | operation);
201
202 __uniphier_cache_sync(data);
203}
204
205static void __uniphier_cache_maint_range(struct uniphier_cache_data *data,
206 unsigned long start, unsigned long end,
207 u32 operation)
208{
209 unsigned long size;
210
211 /*
212 * If the start address is not aligned,
213 * perform a cache operation for the first cache-line
214 */
215 start = start & ~(data->line_size - 1);
216
217 size = end - start;
218
219 if (unlikely(size >= (unsigned long)(-data->line_size))) {
220 /* this means cache operation for all range */
221 __uniphier_cache_maint_all(data, operation);
222 return;
223 }
224
225 /*
226 * If the end address is not aligned,
227 * perform a cache operation for the last cache-line
228 */
229 size = ALIGN(size, data->line_size);
230
231 while (size) {
232 unsigned long chunk_size = min_t(unsigned long, size,
233 data->range_op_max_size);
234
235 __uniphier_cache_maint_common(data, start, chunk_size,
236 UNIPHIER_SSCOQM_S_RANGE | operation);
237
238 start += chunk_size;
239 size -= chunk_size;
240 }
241
242 __uniphier_cache_sync(data);
243}
244
245static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on)
246{
247 u32 val = 0;
248
249 if (on)
250 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON;
251
252 writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC);
253}
254
255static void __init __uniphier_cache_set_locked_ways(
256 struct uniphier_cache_data *data,
257 u32 way_mask)
258{
259 data->way_locked_mask = way_mask & data->way_present_mask;
260
261 writel_relaxed(~data->way_locked_mask & data->way_present_mask,
262 data->ctrl_base + UNIPHIER_SSCLPDAWCR);
263}
264
265static void uniphier_cache_maint_range(unsigned long start, unsigned long end,
266 u32 operation)
267{
268 struct uniphier_cache_data *data;
269
270 list_for_each_entry(data, &uniphier_cache_list, list)
271 __uniphier_cache_maint_range(data, start, end, operation);
272}
273
274static void uniphier_cache_maint_all(u32 operation)
275{
276 struct uniphier_cache_data *data;
277
278 list_for_each_entry(data, &uniphier_cache_list, list)
279 __uniphier_cache_maint_all(data, operation);
280}
281
282static void uniphier_cache_inv_range(unsigned long start, unsigned long end)
283{
284 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV);
285}
286
287static void uniphier_cache_clean_range(unsigned long start, unsigned long end)
288{
289 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN);
290}
291
292static void uniphier_cache_flush_range(unsigned long start, unsigned long end)
293{
294 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH);
295}
296
297static void __init uniphier_cache_inv_all(void)
298{
299 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV);
300}
301
302static void uniphier_cache_flush_all(void)
303{
304 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH);
305}
306
307static void uniphier_cache_disable(void)
308{
309 struct uniphier_cache_data *data;
310
311 list_for_each_entry_reverse(data, &uniphier_cache_list, list)
312 __uniphier_cache_enable(data, false);
313
314 uniphier_cache_flush_all();
315}
316
317static void __init uniphier_cache_enable(void)
318{
319 struct uniphier_cache_data *data;
320
321 uniphier_cache_inv_all();
322
323 list_for_each_entry(data, &uniphier_cache_list, list) {
324 __uniphier_cache_enable(data, true);
325 __uniphier_cache_set_locked_ways(data, 0);
326 }
327}
328
329static void uniphier_cache_sync(void)
330{
331 struct uniphier_cache_data *data;
332
333 list_for_each_entry(data, &uniphier_cache_list, list)
334 __uniphier_cache_sync(data);
335}
336
337int __init uniphier_cache_l2_is_enabled(void)
338{
339 struct uniphier_cache_data *data;
340
341 data = list_first_entry_or_null(&uniphier_cache_list,
342 struct uniphier_cache_data, list);
343 if (!data)
344 return 0;
345
346 return !!(readl_relaxed(data->ctrl_base + UNIPHIER_SSCC) &
347 UNIPHIER_SSCC_ON);
348}
349
350void __init uniphier_cache_l2_touch_range(unsigned long start,
351 unsigned long end)
352{
353 struct uniphier_cache_data *data;
354
355 data = list_first_entry_or_null(&uniphier_cache_list,
356 struct uniphier_cache_data, list);
357 if (data)
358 __uniphier_cache_maint_range(data, start, end,
359 UNIPHIER_SSCOQM_TID_WAY |
360 UNIPHIER_SSCOQM_CM_TOUCH);
361}
362
363void __init uniphier_cache_l2_set_locked_ways(u32 way_mask)
364{
365 struct uniphier_cache_data *data;
366
367 data = list_first_entry_or_null(&uniphier_cache_list,
368 struct uniphier_cache_data, list);
369 if (data)
370 __uniphier_cache_set_locked_ways(data, way_mask);
371}
372
373static const struct of_device_id uniphier_cache_match[] __initconst = {
374 {
375 .compatible = "socionext,uniphier-system-cache",
376 },
377 { /* sentinel */ }
378};
379
380static int __init __uniphier_cache_init(struct device_node *np,
381 unsigned int *cache_level)
382{
383 struct uniphier_cache_data *data;
384 u32 level, cache_size;
385 struct device_node *next_np;
386 int ret = 0;
387
388 if (!of_match_node(uniphier_cache_match, np)) {
389 pr_err("L%d: not compatible with uniphier cache\n",
390 *cache_level);
391 return -EINVAL;
392 }
393
394 if (of_property_read_u32(np, "cache-level", &level)) {
395 pr_err("L%d: cache-level is not specified\n", *cache_level);
396 return -EINVAL;
397 }
398
399 if (level != *cache_level) {
400 pr_err("L%d: cache-level is unexpected value %d\n",
401 *cache_level, level);
402 return -EINVAL;
403 }
404
405 if (!of_property_read_bool(np, "cache-unified")) {
406 pr_err("L%d: cache-unified is not specified\n", *cache_level);
407 return -EINVAL;
408 }
409
410 data = kzalloc(sizeof(*data), GFP_KERNEL);
411 if (!data)
412 return -ENOMEM;
413
414 if (of_property_read_u32(np, "cache-line-size", &data->line_size) ||
415 !is_power_of_2(data->line_size)) {
416 pr_err("L%d: cache-line-size is unspecified or invalid\n",
417 *cache_level);
418 ret = -EINVAL;
419 goto err;
420 }
421
422 if (of_property_read_u32(np, "cache-sets", &data->nsets) ||
423 !is_power_of_2(data->nsets)) {
424 pr_err("L%d: cache-sets is unspecified or invalid\n",
425 *cache_level);
426 ret = -EINVAL;
427 goto err;
428 }
429
430 if (of_property_read_u32(np, "cache-size", &cache_size) ||
431 cache_size == 0 || cache_size % (data->nsets * data->line_size)) {
432 pr_err("L%d: cache-size is unspecified or invalid\n",
433 *cache_level);
434 ret = -EINVAL;
435 goto err;
436 }
437
438 data->way_present_mask =
439 ((u32)1 << cache_size / data->nsets / data->line_size) - 1;
440
441 data->ctrl_base = of_iomap(np, 0);
442 if (!data->ctrl_base) {
443 pr_err("L%d: failed to map control register\n", *cache_level);
444 ret = -ENOMEM;
445 goto err;
446 }
447
448 data->rev_base = of_iomap(np, 1);
449 if (!data->rev_base) {
450 pr_err("L%d: failed to map revision register\n", *cache_level);
451 ret = -ENOMEM;
452 goto err;
453 }
454
455 data->op_base = of_iomap(np, 2);
456 if (!data->op_base) {
457 pr_err("L%d: failed to map operation register\n", *cache_level);
458 ret = -ENOMEM;
459 goto err;
460 }
461
462 if (*cache_level == 2) {
463 u32 revision = readl(data->rev_base + UNIPHIER_SSCID);
464 /*
465 * The size of range operation is limited to (1 << 22) or less
466 * for PH-sLD8 or older SoCs.
467 */
468 if (revision <= 0x16)
469 data->range_op_max_size = (u32)1 << 22;
470 }
471
472 data->range_op_max_size -= data->line_size;
473
474 INIT_LIST_HEAD(&data->list);
475 list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */
476
477 /*
478 * OK, this level has been successfully initialized. Look for the next
479 * level cache. Do not roll back even if the initialization of the
480 * next level cache fails because we want to continue with available
481 * cache levels.
482 */
483 next_np = of_find_next_cache_node(np);
484 if (next_np) {
485 (*cache_level)++;
486 ret = __uniphier_cache_init(next_np, cache_level);
487 }
488 of_node_put(next_np);
489
490 return ret;
491err:
492 iounmap(data->op_base);
493 iounmap(data->rev_base);
494 iounmap(data->ctrl_base);
495 kfree(data);
496
497 return ret;
498}
499
500int __init uniphier_cache_init(void)
501{
502 struct device_node *np = NULL;
503 unsigned int cache_level;
504 int ret = 0;
505
506 /* look for level 2 cache */
507 while ((np = of_find_matching_node(np, uniphier_cache_match)))
508 if (!of_property_read_u32(np, "cache-level", &cache_level) &&
509 cache_level == 2)
510 break;
511
512 if (!np)
513 return -ENODEV;
514
515 ret = __uniphier_cache_init(np, &cache_level);
516 of_node_put(np);
517
518 if (ret) {
519 /*
520 * Error out iif L2 initialization fails. Continue with any
521 * error on L3 or outer because they are optional.
522 */
523 if (cache_level == 2) {
524 pr_err("failed to initialize L2 cache\n");
525 return ret;
526 }
527
528 cache_level--;
529 ret = 0;
530 }
531
532 outer_cache.inv_range = uniphier_cache_inv_range;
533 outer_cache.clean_range = uniphier_cache_clean_range;
534 outer_cache.flush_range = uniphier_cache_flush_range;
535 outer_cache.flush_all = uniphier_cache_flush_all;
536 outer_cache.disable = uniphier_cache_disable;
537 outer_cache.sync = uniphier_cache_sync;
538
539 uniphier_cache_enable();
540
541 pr_info("enabled outer cache (cache level: %d)\n", cache_level);
542
543 return ret;
544}