Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2015-2016 Socionext Inc.
4 * Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5 */
6
7#define pr_fmt(fmt) "uniphier: " fmt
8
9#include <linux/bitops.h>
10#include <linux/init.h>
11#include <linux/io.h>
12#include <linux/log2.h>
13#include <linux/of_address.h>
14#include <linux/slab.h>
15#include <asm/hardware/cache-uniphier.h>
16#include <asm/outercache.h>
17
18/* control registers */
19#define UNIPHIER_SSCC 0x0 /* Control Register */
20#define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */
21#define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */
22#define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */
23#define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */
24#define UNIPHIER_SSCC_ON BIT(0) /* enable cache */
25#define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */
26#define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */
27
28/* revision registers */
29#define UNIPHIER_SSCID 0x0 /* ID Register */
30
31/* operation registers */
32#define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */
33#define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */
34#define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */
35#define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */
36#define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */
37#define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */
38#define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */
39#define UNIPHIER_SSCOQM_S_MASK (0x3 << 17)
40#define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17)
41#define UNIPHIER_SSCOQM_S_ALL (0x1 << 17)
42#define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */
43#define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */
44#define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */
45#define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */
46#define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */
47#define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */
48#define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/
49#define UNIPHIER_SSCOPPQSEF_FE BIT(1)
50#define UNIPHIER_SSCOPPQSEF_OE BIT(0)
51#define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */
52#define UNIPHIER_SSCOLPQS_EF BIT(2)
53#define UNIPHIER_SSCOLPQS_EST BIT(1)
54#define UNIPHIER_SSCOLPQS_QST BIT(0)
55
56/* Is the operation region specified by address range? */
57#define UNIPHIER_SSCOQM_S_IS_RANGE(op) \
58 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE)
59
60/**
61 * uniphier_cache_data - UniPhier outer cache specific data
62 *
63 * @ctrl_base: virtual base address of control registers
64 * @rev_base: virtual base address of revision registers
65 * @op_base: virtual base address of operation registers
66 * @way_mask: each bit specifies if the way is present
67 * @nsets: number of associativity sets
68 * @line_size: line size in bytes
69 * @range_op_max_size: max size that can be handled by a single range operation
70 * @list: list node to include this level in the whole cache hierarchy
71 */
72struct uniphier_cache_data {
73 void __iomem *ctrl_base;
74 void __iomem *rev_base;
75 void __iomem *op_base;
76 void __iomem *way_ctrl_base;
77 u32 way_mask;
78 u32 nsets;
79 u32 line_size;
80 u32 range_op_max_size;
81 struct list_head list;
82};
83
84/*
85 * List of the whole outer cache hierarchy. This list is only modified during
86 * the early boot stage, so no mutex is taken for the access to the list.
87 */
88static LIST_HEAD(uniphier_cache_list);
89
90/**
91 * __uniphier_cache_sync - perform a sync point for a particular cache level
92 *
93 * @data: cache controller specific data
94 */
95static void __uniphier_cache_sync(struct uniphier_cache_data *data)
96{
97 /* This sequence need not be atomic. Do not disable IRQ. */
98 writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC,
99 data->op_base + UNIPHIER_SSCOPE);
100 /* need a read back to confirm */
101 readl_relaxed(data->op_base + UNIPHIER_SSCOPE);
102}
103
104/**
105 * __uniphier_cache_maint_common - run a queue operation for a particular level
106 *
107 * @data: cache controller specific data
108 * @start: start address of range operation (don't care for "all" operation)
109 * @size: data size of range operation (don't care for "all" operation)
110 * @operation: flags to specify the desired cache operation
111 */
112static void __uniphier_cache_maint_common(struct uniphier_cache_data *data,
113 unsigned long start,
114 unsigned long size,
115 u32 operation)
116{
117 unsigned long flags;
118
119 /*
120 * No spin lock is necessary here because:
121 *
122 * [1] This outer cache controller is able to accept maintenance
123 * operations from multiple CPUs at a time in an SMP system; if a
124 * maintenance operation is under way and another operation is issued,
125 * the new one is stored in the queue. The controller performs one
126 * operation after another. If the queue is full, the status register,
127 * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has
128 * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have
129 * different instances for each CPU, i.e. each CPU can track the status
130 * of the maintenance operations triggered by itself.
131 *
132 * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ,
133 * SSCOQWN}, are shared between multiple CPUs, but the hardware still
134 * guarantees the registration sequence is atomic; the write access to
135 * them are arbitrated by the hardware. The first accessor to the
136 * register, UNIPHIER_SSCOQM, holds the access right and it is released
137 * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU
138 * is holding the access right, other CPUs fail to register operations.
139 * One CPU should not hold the access right for a long time, so local
140 * IRQs should be disabled while the following sequence.
141 */
142 local_irq_save(flags);
143
144 /* clear the complete notification flag */
145 writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS);
146
147 do {
148 /* set cache operation */
149 writel_relaxed(UNIPHIER_SSCOQM_CE | operation,
150 data->op_base + UNIPHIER_SSCOQM);
151
152 /* set address range if needed */
153 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) {
154 writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD);
155 writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ);
156 }
157 } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) &
158 (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE)));
159
160 /* wait until the operation is completed */
161 while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) !=
162 UNIPHIER_SSCOLPQS_EF))
163 cpu_relax();
164
165 local_irq_restore(flags);
166}
167
168static void __uniphier_cache_maint_all(struct uniphier_cache_data *data,
169 u32 operation)
170{
171 __uniphier_cache_maint_common(data, 0, 0,
172 UNIPHIER_SSCOQM_S_ALL | operation);
173
174 __uniphier_cache_sync(data);
175}
176
177static void __uniphier_cache_maint_range(struct uniphier_cache_data *data,
178 unsigned long start, unsigned long end,
179 u32 operation)
180{
181 unsigned long size;
182
183 /*
184 * If the start address is not aligned,
185 * perform a cache operation for the first cache-line
186 */
187 start = start & ~(data->line_size - 1);
188
189 size = end - start;
190
191 if (unlikely(size >= (unsigned long)(-data->line_size))) {
192 /* this means cache operation for all range */
193 __uniphier_cache_maint_all(data, operation);
194 return;
195 }
196
197 /*
198 * If the end address is not aligned,
199 * perform a cache operation for the last cache-line
200 */
201 size = ALIGN(size, data->line_size);
202
203 while (size) {
204 unsigned long chunk_size = min_t(unsigned long, size,
205 data->range_op_max_size);
206
207 __uniphier_cache_maint_common(data, start, chunk_size,
208 UNIPHIER_SSCOQM_S_RANGE | operation);
209
210 start += chunk_size;
211 size -= chunk_size;
212 }
213
214 __uniphier_cache_sync(data);
215}
216
217static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on)
218{
219 u32 val = 0;
220
221 if (on)
222 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON;
223
224 writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC);
225}
226
227static void __init __uniphier_cache_set_active_ways(
228 struct uniphier_cache_data *data)
229{
230 unsigned int cpu;
231
232 for_each_possible_cpu(cpu)
233 writel_relaxed(data->way_mask, data->way_ctrl_base + 4 * cpu);
234}
235
236static void uniphier_cache_maint_range(unsigned long start, unsigned long end,
237 u32 operation)
238{
239 struct uniphier_cache_data *data;
240
241 list_for_each_entry(data, &uniphier_cache_list, list)
242 __uniphier_cache_maint_range(data, start, end, operation);
243}
244
245static void uniphier_cache_maint_all(u32 operation)
246{
247 struct uniphier_cache_data *data;
248
249 list_for_each_entry(data, &uniphier_cache_list, list)
250 __uniphier_cache_maint_all(data, operation);
251}
252
253static void uniphier_cache_inv_range(unsigned long start, unsigned long end)
254{
255 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV);
256}
257
258static void uniphier_cache_clean_range(unsigned long start, unsigned long end)
259{
260 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN);
261}
262
263static void uniphier_cache_flush_range(unsigned long start, unsigned long end)
264{
265 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH);
266}
267
268static void __init uniphier_cache_inv_all(void)
269{
270 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV);
271}
272
273static void uniphier_cache_flush_all(void)
274{
275 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH);
276}
277
278static void uniphier_cache_disable(void)
279{
280 struct uniphier_cache_data *data;
281
282 list_for_each_entry_reverse(data, &uniphier_cache_list, list)
283 __uniphier_cache_enable(data, false);
284
285 uniphier_cache_flush_all();
286}
287
288static void __init uniphier_cache_enable(void)
289{
290 struct uniphier_cache_data *data;
291
292 uniphier_cache_inv_all();
293
294 list_for_each_entry(data, &uniphier_cache_list, list) {
295 __uniphier_cache_enable(data, true);
296 __uniphier_cache_set_active_ways(data);
297 }
298}
299
300static void uniphier_cache_sync(void)
301{
302 struct uniphier_cache_data *data;
303
304 list_for_each_entry(data, &uniphier_cache_list, list)
305 __uniphier_cache_sync(data);
306}
307
308static const struct of_device_id uniphier_cache_match[] __initconst = {
309 { .compatible = "socionext,uniphier-system-cache" },
310 { /* sentinel */ }
311};
312
313static int __init __uniphier_cache_init(struct device_node *np,
314 unsigned int *cache_level)
315{
316 struct uniphier_cache_data *data;
317 u32 level, cache_size;
318 struct device_node *next_np;
319 int ret = 0;
320
321 if (!of_match_node(uniphier_cache_match, np)) {
322 pr_err("L%d: not compatible with uniphier cache\n",
323 *cache_level);
324 return -EINVAL;
325 }
326
327 if (of_property_read_u32(np, "cache-level", &level)) {
328 pr_err("L%d: cache-level is not specified\n", *cache_level);
329 return -EINVAL;
330 }
331
332 if (level != *cache_level) {
333 pr_err("L%d: cache-level is unexpected value %d\n",
334 *cache_level, level);
335 return -EINVAL;
336 }
337
338 if (!of_property_read_bool(np, "cache-unified")) {
339 pr_err("L%d: cache-unified is not specified\n", *cache_level);
340 return -EINVAL;
341 }
342
343 data = kzalloc(sizeof(*data), GFP_KERNEL);
344 if (!data)
345 return -ENOMEM;
346
347 if (of_property_read_u32(np, "cache-line-size", &data->line_size) ||
348 !is_power_of_2(data->line_size)) {
349 pr_err("L%d: cache-line-size is unspecified or invalid\n",
350 *cache_level);
351 ret = -EINVAL;
352 goto err;
353 }
354
355 if (of_property_read_u32(np, "cache-sets", &data->nsets) ||
356 !is_power_of_2(data->nsets)) {
357 pr_err("L%d: cache-sets is unspecified or invalid\n",
358 *cache_level);
359 ret = -EINVAL;
360 goto err;
361 }
362
363 if (of_property_read_u32(np, "cache-size", &cache_size) ||
364 cache_size == 0 || cache_size % (data->nsets * data->line_size)) {
365 pr_err("L%d: cache-size is unspecified or invalid\n",
366 *cache_level);
367 ret = -EINVAL;
368 goto err;
369 }
370
371 data->way_mask = GENMASK(cache_size / data->nsets / data->line_size - 1,
372 0);
373
374 data->ctrl_base = of_iomap(np, 0);
375 if (!data->ctrl_base) {
376 pr_err("L%d: failed to map control register\n", *cache_level);
377 ret = -ENOMEM;
378 goto err;
379 }
380
381 data->rev_base = of_iomap(np, 1);
382 if (!data->rev_base) {
383 pr_err("L%d: failed to map revision register\n", *cache_level);
384 ret = -ENOMEM;
385 goto err;
386 }
387
388 data->op_base = of_iomap(np, 2);
389 if (!data->op_base) {
390 pr_err("L%d: failed to map operation register\n", *cache_level);
391 ret = -ENOMEM;
392 goto err;
393 }
394
395 data->way_ctrl_base = data->ctrl_base + 0xc00;
396
397 if (*cache_level == 2) {
398 u32 revision = readl(data->rev_base + UNIPHIER_SSCID);
399 /*
400 * The size of range operation is limited to (1 << 22) or less
401 * for PH-sLD8 or older SoCs.
402 */
403 if (revision <= 0x16)
404 data->range_op_max_size = (u32)1 << 22;
405
406 /*
407 * Unfortunatly, the offset address of active way control base
408 * varies from SoC to SoC.
409 */
410 switch (revision) {
411 case 0x11: /* sLD3 */
412 data->way_ctrl_base = data->ctrl_base + 0x870;
413 break;
414 case 0x12: /* LD4 */
415 case 0x16: /* sld8 */
416 data->way_ctrl_base = data->ctrl_base + 0x840;
417 break;
418 default:
419 break;
420 }
421 }
422
423 data->range_op_max_size -= data->line_size;
424
425 INIT_LIST_HEAD(&data->list);
426 list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */
427
428 /*
429 * OK, this level has been successfully initialized. Look for the next
430 * level cache. Do not roll back even if the initialization of the
431 * next level cache fails because we want to continue with available
432 * cache levels.
433 */
434 next_np = of_find_next_cache_node(np);
435 if (next_np) {
436 (*cache_level)++;
437 ret = __uniphier_cache_init(next_np, cache_level);
438 }
439 of_node_put(next_np);
440
441 return ret;
442err:
443 iounmap(data->op_base);
444 iounmap(data->rev_base);
445 iounmap(data->ctrl_base);
446 kfree(data);
447
448 return ret;
449}
450
451int __init uniphier_cache_init(void)
452{
453 struct device_node *np = NULL;
454 unsigned int cache_level;
455 int ret = 0;
456
457 /* look for level 2 cache */
458 while ((np = of_find_matching_node(np, uniphier_cache_match)))
459 if (!of_property_read_u32(np, "cache-level", &cache_level) &&
460 cache_level == 2)
461 break;
462
463 if (!np)
464 return -ENODEV;
465
466 ret = __uniphier_cache_init(np, &cache_level);
467 of_node_put(np);
468
469 if (ret) {
470 /*
471 * Error out iif L2 initialization fails. Continue with any
472 * error on L3 or outer because they are optional.
473 */
474 if (cache_level == 2) {
475 pr_err("failed to initialize L2 cache\n");
476 return ret;
477 }
478
479 cache_level--;
480 ret = 0;
481 }
482
483 outer_cache.inv_range = uniphier_cache_inv_range;
484 outer_cache.clean_range = uniphier_cache_clean_range;
485 outer_cache.flush_range = uniphier_cache_flush_range;
486 outer_cache.flush_all = uniphier_cache_flush_all;
487 outer_cache.disable = uniphier_cache_disable;
488 outer_cache.sync = uniphier_cache_sync;
489
490 uniphier_cache_enable();
491
492 pr_info("enabled outer cache (cache level: %d)\n", cache_level);
493
494 return ret;
495}
1/*
2 * Copyright (C) 2015-2016 Socionext Inc.
3 * Author: Masahiro Yamada <yamada.masahiro@socionext.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16#define pr_fmt(fmt) "uniphier: " fmt
17
18#include <linux/bitops.h>
19#include <linux/init.h>
20#include <linux/io.h>
21#include <linux/log2.h>
22#include <linux/of_address.h>
23#include <linux/slab.h>
24#include <asm/hardware/cache-uniphier.h>
25#include <asm/outercache.h>
26
27/* control registers */
28#define UNIPHIER_SSCC 0x0 /* Control Register */
29#define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */
30#define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */
31#define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */
32#define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */
33#define UNIPHIER_SSCC_ON BIT(0) /* enable cache */
34#define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */
35#define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */
36
37/* revision registers */
38#define UNIPHIER_SSCID 0x0 /* ID Register */
39
40/* operation registers */
41#define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */
42#define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */
43#define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */
44#define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */
45#define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */
46#define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */
47#define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */
48#define UNIPHIER_SSCOQM_S_MASK (0x3 << 17)
49#define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17)
50#define UNIPHIER_SSCOQM_S_ALL (0x1 << 17)
51#define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */
52#define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */
53#define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */
54#define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */
55#define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */
56#define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */
57#define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/
58#define UNIPHIER_SSCOPPQSEF_FE BIT(1)
59#define UNIPHIER_SSCOPPQSEF_OE BIT(0)
60#define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */
61#define UNIPHIER_SSCOLPQS_EF BIT(2)
62#define UNIPHIER_SSCOLPQS_EST BIT(1)
63#define UNIPHIER_SSCOLPQS_QST BIT(0)
64
65/* Is the operation region specified by address range? */
66#define UNIPHIER_SSCOQM_S_IS_RANGE(op) \
67 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE)
68
69/**
70 * uniphier_cache_data - UniPhier outer cache specific data
71 *
72 * @ctrl_base: virtual base address of control registers
73 * @rev_base: virtual base address of revision registers
74 * @op_base: virtual base address of operation registers
75 * @way_mask: each bit specifies if the way is present
76 * @nsets: number of associativity sets
77 * @line_size: line size in bytes
78 * @range_op_max_size: max size that can be handled by a single range operation
79 * @list: list node to include this level in the whole cache hierarchy
80 */
81struct uniphier_cache_data {
82 void __iomem *ctrl_base;
83 void __iomem *rev_base;
84 void __iomem *op_base;
85 void __iomem *way_ctrl_base;
86 u32 way_mask;
87 u32 nsets;
88 u32 line_size;
89 u32 range_op_max_size;
90 struct list_head list;
91};
92
93/*
94 * List of the whole outer cache hierarchy. This list is only modified during
95 * the early boot stage, so no mutex is taken for the access to the list.
96 */
97static LIST_HEAD(uniphier_cache_list);
98
99/**
100 * __uniphier_cache_sync - perform a sync point for a particular cache level
101 *
102 * @data: cache controller specific data
103 */
104static void __uniphier_cache_sync(struct uniphier_cache_data *data)
105{
106 /* This sequence need not be atomic. Do not disable IRQ. */
107 writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC,
108 data->op_base + UNIPHIER_SSCOPE);
109 /* need a read back to confirm */
110 readl_relaxed(data->op_base + UNIPHIER_SSCOPE);
111}
112
113/**
114 * __uniphier_cache_maint_common - run a queue operation for a particular level
115 *
116 * @data: cache controller specific data
117 * @start: start address of range operation (don't care for "all" operation)
118 * @size: data size of range operation (don't care for "all" operation)
119 * @operation: flags to specify the desired cache operation
120 */
121static void __uniphier_cache_maint_common(struct uniphier_cache_data *data,
122 unsigned long start,
123 unsigned long size,
124 u32 operation)
125{
126 unsigned long flags;
127
128 /*
129 * No spin lock is necessary here because:
130 *
131 * [1] This outer cache controller is able to accept maintenance
132 * operations from multiple CPUs at a time in an SMP system; if a
133 * maintenance operation is under way and another operation is issued,
134 * the new one is stored in the queue. The controller performs one
135 * operation after another. If the queue is full, the status register,
136 * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has
137 * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have
138 * different instances for each CPU, i.e. each CPU can track the status
139 * of the maintenance operations triggered by itself.
140 *
141 * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ,
142 * SSCOQWN}, are shared between multiple CPUs, but the hardware still
143 * guarantees the registration sequence is atomic; the write access to
144 * them are arbitrated by the hardware. The first accessor to the
145 * register, UNIPHIER_SSCOQM, holds the access right and it is released
146 * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU
147 * is holding the access right, other CPUs fail to register operations.
148 * One CPU should not hold the access right for a long time, so local
149 * IRQs should be disabled while the following sequence.
150 */
151 local_irq_save(flags);
152
153 /* clear the complete notification flag */
154 writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS);
155
156 do {
157 /* set cache operation */
158 writel_relaxed(UNIPHIER_SSCOQM_CE | operation,
159 data->op_base + UNIPHIER_SSCOQM);
160
161 /* set address range if needed */
162 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) {
163 writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD);
164 writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ);
165 }
166 } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) &
167 (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE)));
168
169 /* wait until the operation is completed */
170 while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) !=
171 UNIPHIER_SSCOLPQS_EF))
172 cpu_relax();
173
174 local_irq_restore(flags);
175}
176
177static void __uniphier_cache_maint_all(struct uniphier_cache_data *data,
178 u32 operation)
179{
180 __uniphier_cache_maint_common(data, 0, 0,
181 UNIPHIER_SSCOQM_S_ALL | operation);
182
183 __uniphier_cache_sync(data);
184}
185
186static void __uniphier_cache_maint_range(struct uniphier_cache_data *data,
187 unsigned long start, unsigned long end,
188 u32 operation)
189{
190 unsigned long size;
191
192 /*
193 * If the start address is not aligned,
194 * perform a cache operation for the first cache-line
195 */
196 start = start & ~(data->line_size - 1);
197
198 size = end - start;
199
200 if (unlikely(size >= (unsigned long)(-data->line_size))) {
201 /* this means cache operation for all range */
202 __uniphier_cache_maint_all(data, operation);
203 return;
204 }
205
206 /*
207 * If the end address is not aligned,
208 * perform a cache operation for the last cache-line
209 */
210 size = ALIGN(size, data->line_size);
211
212 while (size) {
213 unsigned long chunk_size = min_t(unsigned long, size,
214 data->range_op_max_size);
215
216 __uniphier_cache_maint_common(data, start, chunk_size,
217 UNIPHIER_SSCOQM_S_RANGE | operation);
218
219 start += chunk_size;
220 size -= chunk_size;
221 }
222
223 __uniphier_cache_sync(data);
224}
225
226static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on)
227{
228 u32 val = 0;
229
230 if (on)
231 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON;
232
233 writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC);
234}
235
236static void __init __uniphier_cache_set_active_ways(
237 struct uniphier_cache_data *data)
238{
239 unsigned int cpu;
240
241 for_each_possible_cpu(cpu)
242 writel_relaxed(data->way_mask, data->way_ctrl_base + 4 * cpu);
243}
244
245static void uniphier_cache_maint_range(unsigned long start, unsigned long end,
246 u32 operation)
247{
248 struct uniphier_cache_data *data;
249
250 list_for_each_entry(data, &uniphier_cache_list, list)
251 __uniphier_cache_maint_range(data, start, end, operation);
252}
253
254static void uniphier_cache_maint_all(u32 operation)
255{
256 struct uniphier_cache_data *data;
257
258 list_for_each_entry(data, &uniphier_cache_list, list)
259 __uniphier_cache_maint_all(data, operation);
260}
261
262static void uniphier_cache_inv_range(unsigned long start, unsigned long end)
263{
264 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV);
265}
266
267static void uniphier_cache_clean_range(unsigned long start, unsigned long end)
268{
269 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN);
270}
271
272static void uniphier_cache_flush_range(unsigned long start, unsigned long end)
273{
274 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH);
275}
276
277static void __init uniphier_cache_inv_all(void)
278{
279 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV);
280}
281
282static void uniphier_cache_flush_all(void)
283{
284 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH);
285}
286
287static void uniphier_cache_disable(void)
288{
289 struct uniphier_cache_data *data;
290
291 list_for_each_entry_reverse(data, &uniphier_cache_list, list)
292 __uniphier_cache_enable(data, false);
293
294 uniphier_cache_flush_all();
295}
296
297static void __init uniphier_cache_enable(void)
298{
299 struct uniphier_cache_data *data;
300
301 uniphier_cache_inv_all();
302
303 list_for_each_entry(data, &uniphier_cache_list, list) {
304 __uniphier_cache_enable(data, true);
305 __uniphier_cache_set_active_ways(data);
306 }
307}
308
309static void uniphier_cache_sync(void)
310{
311 struct uniphier_cache_data *data;
312
313 list_for_each_entry(data, &uniphier_cache_list, list)
314 __uniphier_cache_sync(data);
315}
316
317static const struct of_device_id uniphier_cache_match[] __initconst = {
318 { .compatible = "socionext,uniphier-system-cache" },
319 { /* sentinel */ }
320};
321
322static int __init __uniphier_cache_init(struct device_node *np,
323 unsigned int *cache_level)
324{
325 struct uniphier_cache_data *data;
326 u32 level, cache_size;
327 struct device_node *next_np;
328 int ret = 0;
329
330 if (!of_match_node(uniphier_cache_match, np)) {
331 pr_err("L%d: not compatible with uniphier cache\n",
332 *cache_level);
333 return -EINVAL;
334 }
335
336 if (of_property_read_u32(np, "cache-level", &level)) {
337 pr_err("L%d: cache-level is not specified\n", *cache_level);
338 return -EINVAL;
339 }
340
341 if (level != *cache_level) {
342 pr_err("L%d: cache-level is unexpected value %d\n",
343 *cache_level, level);
344 return -EINVAL;
345 }
346
347 if (!of_property_read_bool(np, "cache-unified")) {
348 pr_err("L%d: cache-unified is not specified\n", *cache_level);
349 return -EINVAL;
350 }
351
352 data = kzalloc(sizeof(*data), GFP_KERNEL);
353 if (!data)
354 return -ENOMEM;
355
356 if (of_property_read_u32(np, "cache-line-size", &data->line_size) ||
357 !is_power_of_2(data->line_size)) {
358 pr_err("L%d: cache-line-size is unspecified or invalid\n",
359 *cache_level);
360 ret = -EINVAL;
361 goto err;
362 }
363
364 if (of_property_read_u32(np, "cache-sets", &data->nsets) ||
365 !is_power_of_2(data->nsets)) {
366 pr_err("L%d: cache-sets is unspecified or invalid\n",
367 *cache_level);
368 ret = -EINVAL;
369 goto err;
370 }
371
372 if (of_property_read_u32(np, "cache-size", &cache_size) ||
373 cache_size == 0 || cache_size % (data->nsets * data->line_size)) {
374 pr_err("L%d: cache-size is unspecified or invalid\n",
375 *cache_level);
376 ret = -EINVAL;
377 goto err;
378 }
379
380 data->way_mask = GENMASK(cache_size / data->nsets / data->line_size - 1,
381 0);
382
383 data->ctrl_base = of_iomap(np, 0);
384 if (!data->ctrl_base) {
385 pr_err("L%d: failed to map control register\n", *cache_level);
386 ret = -ENOMEM;
387 goto err;
388 }
389
390 data->rev_base = of_iomap(np, 1);
391 if (!data->rev_base) {
392 pr_err("L%d: failed to map revision register\n", *cache_level);
393 ret = -ENOMEM;
394 goto err;
395 }
396
397 data->op_base = of_iomap(np, 2);
398 if (!data->op_base) {
399 pr_err("L%d: failed to map operation register\n", *cache_level);
400 ret = -ENOMEM;
401 goto err;
402 }
403
404 data->way_ctrl_base = data->ctrl_base + 0xc00;
405
406 if (*cache_level == 2) {
407 u32 revision = readl(data->rev_base + UNIPHIER_SSCID);
408 /*
409 * The size of range operation is limited to (1 << 22) or less
410 * for PH-sLD8 or older SoCs.
411 */
412 if (revision <= 0x16)
413 data->range_op_max_size = (u32)1 << 22;
414
415 /*
416 * Unfortunatly, the offset address of active way control base
417 * varies from SoC to SoC.
418 */
419 switch (revision) {
420 case 0x11: /* sLD3 */
421 data->way_ctrl_base = data->ctrl_base + 0x870;
422 break;
423 case 0x12: /* LD4 */
424 case 0x16: /* sld8 */
425 data->way_ctrl_base = data->ctrl_base + 0x840;
426 break;
427 default:
428 break;
429 }
430 }
431
432 data->range_op_max_size -= data->line_size;
433
434 INIT_LIST_HEAD(&data->list);
435 list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */
436
437 /*
438 * OK, this level has been successfully initialized. Look for the next
439 * level cache. Do not roll back even if the initialization of the
440 * next level cache fails because we want to continue with available
441 * cache levels.
442 */
443 next_np = of_find_next_cache_node(np);
444 if (next_np) {
445 (*cache_level)++;
446 ret = __uniphier_cache_init(next_np, cache_level);
447 }
448 of_node_put(next_np);
449
450 return ret;
451err:
452 iounmap(data->op_base);
453 iounmap(data->rev_base);
454 iounmap(data->ctrl_base);
455 kfree(data);
456
457 return ret;
458}
459
460int __init uniphier_cache_init(void)
461{
462 struct device_node *np = NULL;
463 unsigned int cache_level;
464 int ret = 0;
465
466 /* look for level 2 cache */
467 while ((np = of_find_matching_node(np, uniphier_cache_match)))
468 if (!of_property_read_u32(np, "cache-level", &cache_level) &&
469 cache_level == 2)
470 break;
471
472 if (!np)
473 return -ENODEV;
474
475 ret = __uniphier_cache_init(np, &cache_level);
476 of_node_put(np);
477
478 if (ret) {
479 /*
480 * Error out iif L2 initialization fails. Continue with any
481 * error on L3 or outer because they are optional.
482 */
483 if (cache_level == 2) {
484 pr_err("failed to initialize L2 cache\n");
485 return ret;
486 }
487
488 cache_level--;
489 ret = 0;
490 }
491
492 outer_cache.inv_range = uniphier_cache_inv_range;
493 outer_cache.clean_range = uniphier_cache_clean_range;
494 outer_cache.flush_range = uniphier_cache_flush_range;
495 outer_cache.flush_all = uniphier_cache_flush_all;
496 outer_cache.disable = uniphier_cache_disable;
497 outer_cache.sync = uniphier_cache_sync;
498
499 uniphier_cache_enable();
500
501 pr_info("enabled outer cache (cache level: %d)\n", cache_level);
502
503 return ret;
504}