1/*
   2 * This file is subject to the terms and conditions of the GNU General Public
   3 * License.  See the file "COPYING" in the main directory of this archive
   4 * for more details.
   5 *
   6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
   7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
   8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   9 */
  10#include <linux/cpu_pm.h>
  11#include <linux/hardirq.h>
  12#include <linux/init.h>
  13#include <linux/highmem.h>
  14#include <linux/kernel.h>
  15#include <linux/linkage.h>
  16#include <linux/preempt.h>
  17#include <linux/sched.h>
  18#include <linux/smp.h>
  19#include <linux/mm.h>
  20#include <linux/export.h>
  21#include <linux/bitops.h>
  22#include <linux/dma-map-ops.h> /* for dma_default_coherent */
  23
  24#include <asm/bcache.h>
  25#include <asm/bootinfo.h>
  26#include <asm/cache.h>
  27#include <asm/cacheops.h>
  28#include <asm/cpu.h>
  29#include <asm/cpu-features.h>
  30#include <asm/cpu-type.h>
  31#include <asm/io.h>
  32#include <asm/page.h>
 
  33#include <asm/r4kcache.h>
  34#include <asm/sections.h>
  35#include <asm/mmu_context.h>
  36#include <asm/war.h>
  37#include <asm/cacheflush.h> /* for run_uncached() */
  38#include <asm/traps.h>
  39#include <asm/mips-cps.h>
  40
  41/*
  42 * Bits describing what cache ops an SMP callback function may perform.
  43 *
  44 * R4K_HIT   -	Virtual user or kernel address based cache operations. The
  45 *		active_mm must be checked before using user addresses, falling
  46 *		back to kmap.
  47 * R4K_INDEX -	Index based cache operations.
  48 */
  49
  50#define R4K_HIT		BIT(0)
  51#define R4K_INDEX	BIT(1)
  52
  53/**
  54 * r4k_op_needs_ipi() - Decide if a cache op needs to be done on every core.
  55 * @type:	Type of cache operations (R4K_HIT or R4K_INDEX).
  56 *
  57 * Decides whether a cache op needs to be performed on every core in the system.
  58 * This may change depending on the @type of cache operation, as well as the set
  59 * of online CPUs, so preemption should be disabled by the caller to prevent CPU
  60 * hotplug from changing the result.
  61 *
  62 * Returns:	1 if the cache operation @type should be done on every core in
  63 *		the system.
  64 *		0 if the cache operation @type is globalized and only needs to
  65 *		be performed on a simple CPU.
  66 */
  67static inline bool r4k_op_needs_ipi(unsigned int type)
  68{
  69	/* The MIPS Coherence Manager (CM) globalizes address-based cache ops */
  70	if (type == R4K_HIT && mips_cm_present())
  71		return false;
  72
  73	/*
  74	 * Hardware doesn't globalize the required cache ops, so SMP calls may
  75	 * be needed, but only if there are foreign CPUs (non-siblings with
  76	 * separate caches).
  77	 */
  78	/* cpu_foreign_map[] undeclared when !CONFIG_SMP */
  79#ifdef CONFIG_SMP
  80	return !cpumask_empty(&cpu_foreign_map[0]);
  81#else
  82	return false;
  83#endif
  84}
  85
  86/*
  87 * Special Variant of smp_call_function for use by cache functions:
  88 *
  89 *  o No return value
  90 *  o collapses to normal function call on UP kernels
  91 *  o collapses to normal function call on systems with a single shared
  92 *    primary cache.
  93 *  o doesn't disable interrupts on the local CPU
  94 */
  95static inline void r4k_on_each_cpu(unsigned int type,
  96				   void (*func)(void *info), void *info)
  97{
  98	preempt_disable();
  99	if (r4k_op_needs_ipi(type))
 100		smp_call_function_many(&cpu_foreign_map[smp_processor_id()],
 101				       func, info, 1);
 
 102	func(info);
 103	preempt_enable();
 104}
 105
 
 
 
 
 
 
 106/*
 107 * Must die.
 108 */
 109static unsigned long icache_size __read_mostly;
 110static unsigned long dcache_size __read_mostly;
 111static unsigned long vcache_size __read_mostly;
 112static unsigned long scache_size __read_mostly;
 113
 114/*
 115 * Dummy cache handling routines for machines without boardcaches
 116 */
 117static void cache_noop(void) {}
 118
 119static struct bcache_ops no_sc_ops = {
 120	.bc_enable = (void *)cache_noop,
 121	.bc_disable = (void *)cache_noop,
 122	.bc_wback_inv = (void *)cache_noop,
 123	.bc_inv = (void *)cache_noop
 124};
 125
 126struct bcache_ops *bcops = &no_sc_ops;
 127
 128#define cpu_is_r4600_v1_x()	((read_c0_prid() & 0xfffffff0) == 0x00002010)
 129#define cpu_is_r4600_v2_x()	((read_c0_prid() & 0xfffffff0) == 0x00002020)
 130
 131#define R4600_HIT_CACHEOP_WAR_IMPL					\
 132do {									\
 133	if (IS_ENABLED(CONFIG_WAR_R4600_V2_HIT_CACHEOP) &&		\
 134	    cpu_is_r4600_v2_x())					\
 135		*(volatile unsigned long *)CKSEG1;			\
 136	if (IS_ENABLED(CONFIG_WAR_R4600_V1_HIT_CACHEOP))					\
 137		__asm__ __volatile__("nop;nop;nop;nop");		\
 138} while (0)
 139
 140static void (*r4k_blast_dcache_page)(unsigned long addr);
 141
 142static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
 143{
 144	R4600_HIT_CACHEOP_WAR_IMPL;
 145	blast_dcache32_page(addr);
 146}
 147
 148static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
 149{
 
 150	blast_dcache64_page(addr);
 151}
 152
 153static inline void r4k_blast_dcache_page_dc128(unsigned long addr)
 154{
 155	blast_dcache128_page(addr);
 156}
 157
 158static void r4k_blast_dcache_page_setup(void)
 159{
 160	unsigned long  dc_lsize = cpu_dcache_line_size();
 161
 162	switch (dc_lsize) {
 163	case 0:
 164		r4k_blast_dcache_page = (void *)cache_noop;
 165		break;
 166	case 16:
 167		r4k_blast_dcache_page = blast_dcache16_page;
 168		break;
 169	case 32:
 170		r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
 171		break;
 172	case 64:
 173		r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
 174		break;
 175	case 128:
 176		r4k_blast_dcache_page = r4k_blast_dcache_page_dc128;
 177		break;
 178	default:
 179		break;
 180	}
 181}
 182
 183#ifndef CONFIG_EVA
 184#define r4k_blast_dcache_user_page  r4k_blast_dcache_page
 185#else
 186
 187static void (*r4k_blast_dcache_user_page)(unsigned long addr);
 188
 189static void r4k_blast_dcache_user_page_setup(void)
 190{
 191	unsigned long  dc_lsize = cpu_dcache_line_size();
 192
 193	if (dc_lsize == 0)
 194		r4k_blast_dcache_user_page = (void *)cache_noop;
 195	else if (dc_lsize == 16)
 196		r4k_blast_dcache_user_page = blast_dcache16_user_page;
 197	else if (dc_lsize == 32)
 198		r4k_blast_dcache_user_page = blast_dcache32_user_page;
 199	else if (dc_lsize == 64)
 200		r4k_blast_dcache_user_page = blast_dcache64_user_page;
 201}
 202
 203#endif
 204
 205static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
 206
 207static void r4k_blast_dcache_page_indexed_setup(void)
 208{
 209	unsigned long dc_lsize = cpu_dcache_line_size();
 210
 211	if (dc_lsize == 0)
 212		r4k_blast_dcache_page_indexed = (void *)cache_noop;
 213	else if (dc_lsize == 16)
 214		r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
 215	else if (dc_lsize == 32)
 216		r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
 217	else if (dc_lsize == 64)
 218		r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
 219	else if (dc_lsize == 128)
 220		r4k_blast_dcache_page_indexed = blast_dcache128_page_indexed;
 221}
 222
 223void (* r4k_blast_dcache)(void);
 224EXPORT_SYMBOL(r4k_blast_dcache);
 225
 226static void r4k_blast_dcache_setup(void)
 227{
 228	unsigned long dc_lsize = cpu_dcache_line_size();
 229
 230	if (dc_lsize == 0)
 231		r4k_blast_dcache = (void *)cache_noop;
 232	else if (dc_lsize == 16)
 233		r4k_blast_dcache = blast_dcache16;
 234	else if (dc_lsize == 32)
 235		r4k_blast_dcache = blast_dcache32;
 236	else if (dc_lsize == 64)
 237		r4k_blast_dcache = blast_dcache64;
 238	else if (dc_lsize == 128)
 239		r4k_blast_dcache = blast_dcache128;
 240}
 241
 242/* force code alignment (used for CONFIG_WAR_TX49XX_ICACHE_INDEX_INV) */
 243#define JUMP_TO_ALIGN(order) \
 244	__asm__ __volatile__( \
 245		"b\t1f\n\t" \
 246		".align\t" #order "\n\t" \
 247		"1:\n\t" \
 248		)
 249#define CACHE32_UNROLL32_ALIGN	JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
 250#define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
 251
 252static inline void blast_r4600_v1_icache32(void)
 253{
 254	unsigned long flags;
 255
 256	local_irq_save(flags);
 257	blast_icache32();
 258	local_irq_restore(flags);
 259}
 260
 261static inline void tx49_blast_icache32(void)
 262{
 263	unsigned long start = INDEX_BASE;
 264	unsigned long end = start + current_cpu_data.icache.waysize;
 265	unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
 266	unsigned long ws_end = current_cpu_data.icache.ways <<
 267			       current_cpu_data.icache.waybit;
 268	unsigned long ws, addr;
 269
 270	CACHE32_UNROLL32_ALIGN2;
 271	/* I'm in even chunk.  blast odd chunks */
 272	for (ws = 0; ws < ws_end; ws += ws_inc)
 273		for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
 274			cache_unroll(32, kernel_cache, Index_Invalidate_I,
 275				     addr | ws, 32);
 276	CACHE32_UNROLL32_ALIGN;
 277	/* I'm in odd chunk.  blast even chunks */
 278	for (ws = 0; ws < ws_end; ws += ws_inc)
 279		for (addr = start; addr < end; addr += 0x400 * 2)
 280			cache_unroll(32, kernel_cache, Index_Invalidate_I,
 281				     addr | ws, 32);
 282}
 283
 284static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
 285{
 286	unsigned long flags;
 287
 288	local_irq_save(flags);
 289	blast_icache32_page_indexed(page);
 290	local_irq_restore(flags);
 291}
 292
 293static inline void tx49_blast_icache32_page_indexed(unsigned long page)
 294{
 295	unsigned long indexmask = current_cpu_data.icache.waysize - 1;
 296	unsigned long start = INDEX_BASE + (page & indexmask);
 297	unsigned long end = start + PAGE_SIZE;
 298	unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
 299	unsigned long ws_end = current_cpu_data.icache.ways <<
 300			       current_cpu_data.icache.waybit;
 301	unsigned long ws, addr;
 302
 303	CACHE32_UNROLL32_ALIGN2;
 304	/* I'm in even chunk.  blast odd chunks */
 305	for (ws = 0; ws < ws_end; ws += ws_inc)
 306		for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
 307			cache_unroll(32, kernel_cache, Index_Invalidate_I,
 308				     addr | ws, 32);
 309	CACHE32_UNROLL32_ALIGN;
 310	/* I'm in odd chunk.  blast even chunks */
 311	for (ws = 0; ws < ws_end; ws += ws_inc)
 312		for (addr = start; addr < end; addr += 0x400 * 2)
 313			cache_unroll(32, kernel_cache, Index_Invalidate_I,
 314				     addr | ws, 32);
 315}
 316
 317static void (* r4k_blast_icache_page)(unsigned long addr);
 318
 319static void r4k_blast_icache_page_setup(void)
 320{
 321	unsigned long ic_lsize = cpu_icache_line_size();
 322
 323	if (ic_lsize == 0)
 324		r4k_blast_icache_page = (void *)cache_noop;
 325	else if (ic_lsize == 16)
 326		r4k_blast_icache_page = blast_icache16_page;
 327	else if (ic_lsize == 32 && current_cpu_type() == CPU_LOONGSON2EF)
 328		r4k_blast_icache_page = loongson2_blast_icache32_page;
 329	else if (ic_lsize == 32)
 330		r4k_blast_icache_page = blast_icache32_page;
 331	else if (ic_lsize == 64)
 332		r4k_blast_icache_page = blast_icache64_page;
 333	else if (ic_lsize == 128)
 334		r4k_blast_icache_page = blast_icache128_page;
 335}
 336
 337#ifndef CONFIG_EVA
 338#define r4k_blast_icache_user_page  r4k_blast_icache_page
 339#else
 340
 341static void (*r4k_blast_icache_user_page)(unsigned long addr);
 342
 343static void r4k_blast_icache_user_page_setup(void)
 344{
 345	unsigned long ic_lsize = cpu_icache_line_size();
 346
 347	if (ic_lsize == 0)
 348		r4k_blast_icache_user_page = (void *)cache_noop;
 349	else if (ic_lsize == 16)
 350		r4k_blast_icache_user_page = blast_icache16_user_page;
 351	else if (ic_lsize == 32)
 352		r4k_blast_icache_user_page = blast_icache32_user_page;
 353	else if (ic_lsize == 64)
 354		r4k_blast_icache_user_page = blast_icache64_user_page;
 355}
 356
 357#endif
 358
 359static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
 360
 361static void r4k_blast_icache_page_indexed_setup(void)
 362{
 363	unsigned long ic_lsize = cpu_icache_line_size();
 364
 365	if (ic_lsize == 0)
 366		r4k_blast_icache_page_indexed = (void *)cache_noop;
 367	else if (ic_lsize == 16)
 368		r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
 369	else if (ic_lsize == 32) {
 370		if (IS_ENABLED(CONFIG_WAR_R4600_V1_INDEX_ICACHEOP) &&
 371		    cpu_is_r4600_v1_x())
 372			r4k_blast_icache_page_indexed =
 373				blast_icache32_r4600_v1_page_indexed;
 374		else if (IS_ENABLED(CONFIG_WAR_TX49XX_ICACHE_INDEX_INV))
 375			r4k_blast_icache_page_indexed =
 376				tx49_blast_icache32_page_indexed;
 377		else if (current_cpu_type() == CPU_LOONGSON2EF)
 378			r4k_blast_icache_page_indexed =
 379				loongson2_blast_icache32_page_indexed;
 380		else
 381			r4k_blast_icache_page_indexed =
 382				blast_icache32_page_indexed;
 383	} else if (ic_lsize == 64)
 384		r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
 385}
 386
 387void (* r4k_blast_icache)(void);
 388EXPORT_SYMBOL(r4k_blast_icache);
 389
 390static void r4k_blast_icache_setup(void)
 391{
 392	unsigned long ic_lsize = cpu_icache_line_size();
 393
 394	if (ic_lsize == 0)
 395		r4k_blast_icache = (void *)cache_noop;
 396	else if (ic_lsize == 16)
 397		r4k_blast_icache = blast_icache16;
 398	else if (ic_lsize == 32) {
 399		if (IS_ENABLED(CONFIG_WAR_R4600_V1_INDEX_ICACHEOP) &&
 400		    cpu_is_r4600_v1_x())
 401			r4k_blast_icache = blast_r4600_v1_icache32;
 402		else if (IS_ENABLED(CONFIG_WAR_TX49XX_ICACHE_INDEX_INV))
 403			r4k_blast_icache = tx49_blast_icache32;
 404		else if (current_cpu_type() == CPU_LOONGSON2EF)
 405			r4k_blast_icache = loongson2_blast_icache32;
 406		else
 407			r4k_blast_icache = blast_icache32;
 408	} else if (ic_lsize == 64)
 409		r4k_blast_icache = blast_icache64;
 410	else if (ic_lsize == 128)
 411		r4k_blast_icache = blast_icache128;
 412}
 413
 414static void (* r4k_blast_scache_page)(unsigned long addr);
 415
 416static void r4k_blast_scache_page_setup(void)
 417{
 418	unsigned long sc_lsize = cpu_scache_line_size();
 419
 420	if (scache_size == 0)
 421		r4k_blast_scache_page = (void *)cache_noop;
 422	else if (sc_lsize == 16)
 423		r4k_blast_scache_page = blast_scache16_page;
 424	else if (sc_lsize == 32)
 425		r4k_blast_scache_page = blast_scache32_page;
 426	else if (sc_lsize == 64)
 427		r4k_blast_scache_page = blast_scache64_page;
 428	else if (sc_lsize == 128)
 429		r4k_blast_scache_page = blast_scache128_page;
 430}
 431
 432static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
 433
 434static void r4k_blast_scache_page_indexed_setup(void)
 435{
 436	unsigned long sc_lsize = cpu_scache_line_size();
 437
 438	if (scache_size == 0)
 439		r4k_blast_scache_page_indexed = (void *)cache_noop;
 440	else if (sc_lsize == 16)
 441		r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
 442	else if (sc_lsize == 32)
 443		r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
 444	else if (sc_lsize == 64)
 445		r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
 446	else if (sc_lsize == 128)
 447		r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
 448}
 449
 450static void (* r4k_blast_scache)(void);
 451
 452static void r4k_blast_scache_setup(void)
 453{
 454	unsigned long sc_lsize = cpu_scache_line_size();
 455
 456	if (scache_size == 0)
 457		r4k_blast_scache = (void *)cache_noop;
 458	else if (sc_lsize == 16)
 459		r4k_blast_scache = blast_scache16;
 460	else if (sc_lsize == 32)
 461		r4k_blast_scache = blast_scache32;
 462	else if (sc_lsize == 64)
 463		r4k_blast_scache = blast_scache64;
 464	else if (sc_lsize == 128)
 465		r4k_blast_scache = blast_scache128;
 466}
 467
 468static void (*r4k_blast_scache_node)(long node);
 469
 470static void r4k_blast_scache_node_setup(void)
 471{
 472	unsigned long sc_lsize = cpu_scache_line_size();
 473
 474	if (current_cpu_type() != CPU_LOONGSON64)
 475		r4k_blast_scache_node = (void *)cache_noop;
 476	else if (sc_lsize == 16)
 477		r4k_blast_scache_node = blast_scache16_node;
 478	else if (sc_lsize == 32)
 479		r4k_blast_scache_node = blast_scache32_node;
 480	else if (sc_lsize == 64)
 481		r4k_blast_scache_node = blast_scache64_node;
 482	else if (sc_lsize == 128)
 483		r4k_blast_scache_node = blast_scache128_node;
 484}
 485
 486static inline void local_r4k___flush_cache_all(void * args)
 487{
 
 
 
 
 
 
 
 488	switch (current_cpu_type()) {
 489	case CPU_LOONGSON2EF:
 490	case CPU_R4000SC:
 491	case CPU_R4000MC:
 492	case CPU_R4400SC:
 493	case CPU_R4400MC:
 494	case CPU_R10000:
 495	case CPU_R12000:
 496	case CPU_R14000:
 497	case CPU_R16000:
 498		/*
 499		 * These caches are inclusive caches, that is, if something
 500		 * is not cached in the S-cache, we know it also won't be
 501		 * in one of the primary caches.
 502		 */
 503		r4k_blast_scache();
 504		break;
 505
 506	case CPU_LOONGSON64:
 507		/* Use get_ebase_cpunum() for both NUMA=y/n */
 508		r4k_blast_scache_node(get_ebase_cpunum() >> 2);
 509		break;
 510
 511	case CPU_BMIPS5000:
 512		r4k_blast_scache();
 513		__sync();
 514		break;
 515
 516	default:
 517		r4k_blast_dcache();
 518		r4k_blast_icache();
 519		break;
 520	}
 521}
 522
 523static void r4k___flush_cache_all(void)
 524{
 525	r4k_on_each_cpu(R4K_INDEX, local_r4k___flush_cache_all, NULL);
 526}
 527
 528/**
 529 * has_valid_asid() - Determine if an mm already has an ASID.
 530 * @mm:		Memory map.
 531 * @type:	R4K_HIT or R4K_INDEX, type of cache op.
 532 *
 533 * Determines whether @mm already has an ASID on any of the CPUs which cache ops
 534 * of type @type within an r4k_on_each_cpu() call will affect. If
 535 * r4k_on_each_cpu() does an SMP call to a single VPE in each core, then the
 536 * scope of the operation is confined to sibling CPUs, otherwise all online CPUs
 537 * will need to be checked.
 538 *
 539 * Must be called in non-preemptive context.
 540 *
 541 * Returns:	1 if the CPUs affected by @type cache ops have an ASID for @mm.
 542 *		0 otherwise.
 543 */
 544static inline int has_valid_asid(const struct mm_struct *mm, unsigned int type)
 545{
 546	unsigned int i;
 547	const cpumask_t *mask = cpu_present_mask;
 548
 549	if (cpu_has_mmid)
 550		return cpu_context(0, mm) != 0;
 551
 552	/* cpu_sibling_map[] undeclared when !CONFIG_SMP */
 553#ifdef CONFIG_SMP
 554	/*
 555	 * If r4k_on_each_cpu does SMP calls, it does them to a single VPE in
 556	 * each foreign core, so we only need to worry about siblings.
 557	 * Otherwise we need to worry about all present CPUs.
 558	 */
 559	if (r4k_op_needs_ipi(type))
 560		mask = &cpu_sibling_map[smp_processor_id()];
 561#endif
 562	for_each_cpu(i, mask)
 563		if (cpu_context(i, mm))
 564			return 1;
 
 565	return 0;
 
 
 
 566}
 567
 568static void r4k__flush_cache_vmap(void)
 569{
 570	r4k_blast_dcache();
 571}
 572
 573static void r4k__flush_cache_vunmap(void)
 574{
 575	r4k_blast_dcache();
 576}
 577
 578/*
 579 * Note: flush_tlb_range() assumes flush_cache_range() sufficiently flushes
 580 * whole caches when vma is executable.
 581 */
 582static inline void local_r4k_flush_cache_range(void * args)
 583{
 584	struct vm_area_struct *vma = args;
 585	int exec = vma->vm_flags & VM_EXEC;
 586
 587	if (!has_valid_asid(vma->vm_mm, R4K_INDEX))
 588		return;
 589
 590	/*
 591	 * If dcache can alias, we must blast it since mapping is changing.
 592	 * If executable, we must ensure any dirty lines are written back far
 593	 * enough to be visible to icache.
 594	 */
 595	if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
 596		r4k_blast_dcache();
 597	/* If executable, blast stale lines from icache */
 598	if (exec)
 599		r4k_blast_icache();
 600}
 601
 602static void r4k_flush_cache_range(struct vm_area_struct *vma,
 603	unsigned long start, unsigned long end)
 604{
 605	int exec = vma->vm_flags & VM_EXEC;
 606
 607	if (cpu_has_dc_aliases || exec)
 608		r4k_on_each_cpu(R4K_INDEX, local_r4k_flush_cache_range, vma);
 609}
 610
 611static inline void local_r4k_flush_cache_mm(void * args)
 612{
 613	struct mm_struct *mm = args;
 614
 615	if (!has_valid_asid(mm, R4K_INDEX))
 616		return;
 617
 618	/*
 619	 * Kludge alert.  For obscure reasons R4000SC and R4400SC go nuts if we
 620	 * only flush the primary caches but R1x000 behave sane ...
 621	 * R4000SC and R4400SC indexed S-cache ops also invalidate primary
 622	 * caches, so we can bail out early.
 623	 */
 624	if (current_cpu_type() == CPU_R4000SC ||
 625	    current_cpu_type() == CPU_R4000MC ||
 626	    current_cpu_type() == CPU_R4400SC ||
 627	    current_cpu_type() == CPU_R4400MC) {
 628		r4k_blast_scache();
 629		return;
 630	}
 631
 632	r4k_blast_dcache();
 633}
 634
 635static void r4k_flush_cache_mm(struct mm_struct *mm)
 636{
 637	if (!cpu_has_dc_aliases)
 638		return;
 639
 640	r4k_on_each_cpu(R4K_INDEX, local_r4k_flush_cache_mm, mm);
 641}
 642
 643struct flush_cache_page_args {
 644	struct vm_area_struct *vma;
 645	unsigned long addr;
 646	unsigned long pfn;
 647};
 648
 649static inline void local_r4k_flush_cache_page(void *args)
 650{
 651	struct flush_cache_page_args *fcp_args = args;
 652	struct vm_area_struct *vma = fcp_args->vma;
 653	unsigned long addr = fcp_args->addr;
 654	struct page *page = pfn_to_page(fcp_args->pfn);
 655	int exec = vma->vm_flags & VM_EXEC;
 656	struct mm_struct *mm = vma->vm_mm;
 657	int map_coherent = 0;
 
 
 658	pmd_t *pmdp;
 659	pte_t *ptep;
 660	void *vaddr;
 661
 662	/*
 663	 * If owns no valid ASID yet, cannot possibly have gotten
 664	 * this page into the cache.
 665	 */
 666	if (!has_valid_asid(mm, R4K_HIT))
 667		return;
 668
 669	addr &= PAGE_MASK;
 670	pmdp = pmd_off(mm, addr);
 671	ptep = pte_offset_kernel(pmdp, addr);
 
 
 672
 673	/*
 674	 * If the page isn't marked valid, the page cannot possibly be
 675	 * in the cache.
 676	 */
 677	if (!(pte_present(*ptep)))
 678		return;
 679
 680	if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
 681		vaddr = NULL;
 682	else {
 683		/*
 684		 * Use kmap_coherent or kmap_atomic to do flushes for
 685		 * another ASID than the current one.
 686		 */
 687		map_coherent = (cpu_has_dc_aliases &&
 688				page_mapcount(page) &&
 689				!Page_dcache_dirty(page));
 690		if (map_coherent)
 691			vaddr = kmap_coherent(page, addr);
 692		else
 693			vaddr = kmap_atomic(page);
 694		addr = (unsigned long)vaddr;
 695	}
 696
 697	if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
 698		vaddr ? r4k_blast_dcache_page(addr) :
 699			r4k_blast_dcache_user_page(addr);
 700		if (exec && !cpu_icache_snoops_remote_store)
 701			r4k_blast_scache_page(addr);
 702	}
 703	if (exec) {
 704		if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) {
 705			drop_mmu_context(mm);
 
 
 
 706		} else
 707			vaddr ? r4k_blast_icache_page(addr) :
 708				r4k_blast_icache_user_page(addr);
 709	}
 710
 711	if (vaddr) {
 712		if (map_coherent)
 713			kunmap_coherent();
 714		else
 715			kunmap_atomic(vaddr);
 716	}
 717}
 718
 719static void r4k_flush_cache_page(struct vm_area_struct *vma,
 720	unsigned long addr, unsigned long pfn)
 721{
 722	struct flush_cache_page_args args;
 723
 724	args.vma = vma;
 725	args.addr = addr;
 726	args.pfn = pfn;
 727
 728	r4k_on_each_cpu(R4K_HIT, local_r4k_flush_cache_page, &args);
 729}
 730
 731static inline void local_r4k_flush_data_cache_page(void * addr)
 732{
 733	r4k_blast_dcache_page((unsigned long) addr);
 734}
 735
 736static void r4k_flush_data_cache_page(unsigned long addr)
 737{
 738	if (in_atomic())
 739		local_r4k_flush_data_cache_page((void *)addr);
 740	else
 741		r4k_on_each_cpu(R4K_HIT, local_r4k_flush_data_cache_page,
 742				(void *) addr);
 743}
 744
 745struct flush_icache_range_args {
 746	unsigned long start;
 747	unsigned long end;
 748	unsigned int type;
 749	bool user;
 750};
 751
 752static inline void __local_r4k_flush_icache_range(unsigned long start,
 753						  unsigned long end,
 754						  unsigned int type,
 755						  bool user)
 756{
 757	if (!cpu_has_ic_fills_f_dc) {
 758		if (type == R4K_INDEX ||
 759		    (type & R4K_INDEX && end - start >= dcache_size)) {
 760			r4k_blast_dcache();
 761		} else {
 762			R4600_HIT_CACHEOP_WAR_IMPL;
 763			if (user)
 764				protected_blast_dcache_range(start, end);
 765			else
 766				blast_dcache_range(start, end);
 767		}
 768	}
 769
 770	if (type == R4K_INDEX ||
 771	    (type & R4K_INDEX && end - start > icache_size))
 772		r4k_blast_icache();
 773	else {
 774		switch (boot_cpu_type()) {
 775		case CPU_LOONGSON2EF:
 776			protected_loongson2_blast_icache_range(start, end);
 777			break;
 778
 779		default:
 780			if (user)
 781				protected_blast_icache_range(start, end);
 782			else
 783				blast_icache_range(start, end);
 784			break;
 785		}
 786	}
 787}
 788
 789static inline void local_r4k_flush_icache_range(unsigned long start,
 790						unsigned long end)
 791{
 792	__local_r4k_flush_icache_range(start, end, R4K_HIT | R4K_INDEX, false);
 793}
 794
 795static inline void local_r4k_flush_icache_user_range(unsigned long start,
 796						     unsigned long end)
 797{
 798	__local_r4k_flush_icache_range(start, end, R4K_HIT | R4K_INDEX, true);
 799}
 800
 801static inline void local_r4k_flush_icache_range_ipi(void *args)
 802{
 803	struct flush_icache_range_args *fir_args = args;
 804	unsigned long start = fir_args->start;
 805	unsigned long end = fir_args->end;
 806	unsigned int type = fir_args->type;
 807	bool user = fir_args->user;
 808
 809	__local_r4k_flush_icache_range(start, end, type, user);
 810}
 811
 812static void __r4k_flush_icache_range(unsigned long start, unsigned long end,
 813				     bool user)
 814{
 815	struct flush_icache_range_args args;
 816	unsigned long size, cache_size;
 817
 818	args.start = start;
 819	args.end = end;
 820	args.type = R4K_HIT | R4K_INDEX;
 821	args.user = user;
 822
 823	/*
 824	 * Indexed cache ops require an SMP call.
 825	 * Consider if that can or should be avoided.
 826	 */
 827	preempt_disable();
 828	if (r4k_op_needs_ipi(R4K_INDEX) && !r4k_op_needs_ipi(R4K_HIT)) {
 829		/*
 830		 * If address-based cache ops don't require an SMP call, then
 831		 * use them exclusively for small flushes.
 832		 */
 833		size = end - start;
 834		cache_size = icache_size;
 835		if (!cpu_has_ic_fills_f_dc) {
 836			size *= 2;
 837			cache_size += dcache_size;
 838		}
 839		if (size <= cache_size)
 840			args.type &= ~R4K_INDEX;
 841	}
 842	r4k_on_each_cpu(args.type, local_r4k_flush_icache_range_ipi, &args);
 843	preempt_enable();
 844	instruction_hazard();
 845}
 846
 847static void r4k_flush_icache_range(unsigned long start, unsigned long end)
 848{
 849	return __r4k_flush_icache_range(start, end, false);
 850}
 851
 852static void r4k_flush_icache_user_range(unsigned long start, unsigned long end)
 853{
 854	return __r4k_flush_icache_range(start, end, true);
 855}
 856
 857#ifdef CONFIG_DMA_NONCOHERENT
 858
 859static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
 860{
 861	/* Catch bad driver code */
 862	if (WARN_ON(size == 0))
 863		return;
 864
 865	preempt_disable();
 866	if (cpu_has_inclusive_pcaches) {
 867		if (size >= scache_size) {
 868			if (current_cpu_type() != CPU_LOONGSON64)
 869				r4k_blast_scache();
 870			else
 871				r4k_blast_scache_node(pa_to_nid(addr));
 872		} else {
 873			blast_scache_range(addr, addr + size);
 874		}
 875		preempt_enable();
 876		__sync();
 877		return;
 878	}
 879
 880	/*
 881	 * Either no secondary cache or the available caches don't have the
 882	 * subset property so we have to flush the primary caches
 883	 * explicitly.
 884	 * If we would need IPI to perform an INDEX-type operation, then
 885	 * we have to use the HIT-type alternative as IPI cannot be used
 886	 * here due to interrupts possibly being disabled.
 887	 */
 888	if (!r4k_op_needs_ipi(R4K_INDEX) && size >= dcache_size) {
 889		r4k_blast_dcache();
 890	} else {
 891		R4600_HIT_CACHEOP_WAR_IMPL;
 892		blast_dcache_range(addr, addr + size);
 893	}
 894	preempt_enable();
 895
 896	bc_wback_inv(addr, size);
 897	__sync();
 898}
 899
 900static void prefetch_cache_inv(unsigned long addr, unsigned long size)
 901{
 902	unsigned int linesz = cpu_scache_line_size();
 903	unsigned long addr0 = addr, addr1;
 904
 905	addr0 &= ~(linesz - 1);
 906	addr1 = (addr0 + size - 1) & ~(linesz - 1);
 907
 908	protected_writeback_scache_line(addr0);
 909	if (likely(addr1 != addr0))
 910		protected_writeback_scache_line(addr1);
 911	else
 912		return;
 913
 914	addr0 += linesz;
 915	if (likely(addr1 != addr0))
 916		protected_writeback_scache_line(addr0);
 917	else
 918		return;
 919
 920	addr1 -= linesz;
 921	if (likely(addr1 > addr0))
 922		protected_writeback_scache_line(addr0);
 923}
 924
 925static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
 926{
 927	/* Catch bad driver code */
 928	if (WARN_ON(size == 0))
 929		return;
 930
 931	preempt_disable();
 932
 933	if (current_cpu_type() == CPU_BMIPS5000)
 934		prefetch_cache_inv(addr, size);
 935
 936	if (cpu_has_inclusive_pcaches) {
 937		if (size >= scache_size) {
 938			if (current_cpu_type() != CPU_LOONGSON64)
 939				r4k_blast_scache();
 940			else
 941				r4k_blast_scache_node(pa_to_nid(addr));
 942		} else {
 943			/*
 944			 * There is no clearly documented alignment requirement
 945			 * for the cache instruction on MIPS processors and
 946			 * some processors, among them the RM5200 and RM7000
 947			 * QED processors will throw an address error for cache
 948			 * hit ops with insufficient alignment.	 Solved by
 949			 * aligning the address to cache line size.
 950			 */
 
 
 
 951			blast_inv_scache_range(addr, addr + size);
 952		}
 953		preempt_enable();
 954		__sync();
 955		return;
 956	}
 957
 958	if (!r4k_op_needs_ipi(R4K_INDEX) && size >= dcache_size) {
 959		r4k_blast_dcache();
 960	} else {
 
 
 
 961		R4600_HIT_CACHEOP_WAR_IMPL;
 
 
 962		blast_inv_dcache_range(addr, addr + size);
 963	}
 964	preempt_enable();
 965
 966	bc_inv(addr, size);
 967	__sync();
 968}
 969#endif /* CONFIG_DMA_NONCOHERENT */
 970
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 971static void r4k_flush_icache_all(void)
 972{
 973	if (cpu_has_vtag_icache)
 974		r4k_blast_icache();
 975}
 976
 977struct flush_kernel_vmap_range_args {
 978	unsigned long	vaddr;
 979	int		size;
 980};
 981
 982static inline void local_r4k_flush_kernel_vmap_range_index(void *args)
 983{
 984	/*
 985	 * Aliases only affect the primary caches so don't bother with
 986	 * S-caches or T-caches.
 987	 */
 988	r4k_blast_dcache();
 989}
 990
 991static inline void local_r4k_flush_kernel_vmap_range(void *args)
 992{
 993	struct flush_kernel_vmap_range_args *vmra = args;
 994	unsigned long vaddr = vmra->vaddr;
 995	int size = vmra->size;
 996
 997	/*
 998	 * Aliases only affect the primary caches so don't bother with
 999	 * S-caches or T-caches.
1000	 */
1001	R4600_HIT_CACHEOP_WAR_IMPL;
1002	blast_dcache_range(vaddr, vaddr + size);
 
 
 
 
1003}
1004
1005static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size)
1006{
1007	struct flush_kernel_vmap_range_args args;
1008
1009	args.vaddr = (unsigned long) vaddr;
1010	args.size = size;
1011
1012	if (size >= dcache_size)
1013		r4k_on_each_cpu(R4K_INDEX,
1014				local_r4k_flush_kernel_vmap_range_index, NULL);
1015	else
1016		r4k_on_each_cpu(R4K_HIT, local_r4k_flush_kernel_vmap_range,
1017				&args);
1018}
1019
1020static inline void rm7k_erratum31(void)
1021{
1022	const unsigned long ic_lsize = 32;
1023	unsigned long addr;
1024
1025	/* RM7000 erratum #31. The icache is screwed at startup. */
1026	write_c0_taglo(0);
1027	write_c0_taghi(0);
1028
1029	for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
1030		__asm__ __volatile__ (
1031			".set push\n\t"
1032			".set noreorder\n\t"
1033			".set mips3\n\t"
1034			"cache\t%1, 0(%0)\n\t"
1035			"cache\t%1, 0x1000(%0)\n\t"
1036			"cache\t%1, 0x2000(%0)\n\t"
1037			"cache\t%1, 0x3000(%0)\n\t"
1038			"cache\t%2, 0(%0)\n\t"
1039			"cache\t%2, 0x1000(%0)\n\t"
1040			"cache\t%2, 0x2000(%0)\n\t"
1041			"cache\t%2, 0x3000(%0)\n\t"
1042			"cache\t%1, 0(%0)\n\t"
1043			"cache\t%1, 0x1000(%0)\n\t"
1044			"cache\t%1, 0x2000(%0)\n\t"
1045			"cache\t%1, 0x3000(%0)\n\t"
1046			".set pop\n"
1047			:
1048			: "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill_I));
1049	}
1050}
1051
1052static inline int alias_74k_erratum(struct cpuinfo_mips *c)
1053{
1054	unsigned int imp = c->processor_id & PRID_IMP_MASK;
1055	unsigned int rev = c->processor_id & PRID_REV_MASK;
1056	int present = 0;
1057
1058	/*
1059	 * Early versions of the 74K do not update the cache tags on a
1060	 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG
1061	 * aliases.  In this case it is better to treat the cache as always
1062	 * having aliases.  Also disable the synonym tag update feature
1063	 * where available.  In this case no opportunistic tag update will
1064	 * happen where a load causes a virtual address miss but a physical
1065	 * address hit during a D-cache look-up.
1066	 */
1067	switch (imp) {
1068	case PRID_IMP_74K:
1069		if (rev <= PRID_REV_ENCODE_332(2, 4, 0))
1070			present = 1;
1071		if (rev == PRID_REV_ENCODE_332(2, 4, 0))
1072			write_c0_config6(read_c0_config6() | MTI_CONF6_SYND);
1073		break;
1074	case PRID_IMP_1074K:
1075		if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) {
1076			present = 1;
1077			write_c0_config6(read_c0_config6() | MTI_CONF6_SYND);
1078		}
1079		break;
1080	default:
1081		BUG();
1082	}
1083
1084	return present;
1085}
1086
1087static void b5k_instruction_hazard(void)
1088{
1089	__sync();
1090	__sync();
1091	__asm__ __volatile__(
1092	"       nop; nop; nop; nop; nop; nop; nop; nop\n"
1093	"       nop; nop; nop; nop; nop; nop; nop; nop\n"
1094	"       nop; nop; nop; nop; nop; nop; nop; nop\n"
1095	"       nop; nop; nop; nop; nop; nop; nop; nop\n"
1096	: : : "memory");
1097}
1098
1099static char *way_string[] = { NULL, "direct mapped", "2-way",
1100	"3-way", "4-way", "5-way", "6-way", "7-way", "8-way",
1101	"9-way", "10-way", "11-way", "12-way",
1102	"13-way", "14-way", "15-way", "16-way",
1103};
1104
1105static void probe_pcache(void)
1106{
1107	struct cpuinfo_mips *c = &current_cpu_data;
1108	unsigned int config = read_c0_config();
1109	unsigned int prid = read_c0_prid();
1110	int has_74k_erratum = 0;
1111	unsigned long config1;
1112	unsigned int lsize;
1113
1114	switch (current_cpu_type()) {
1115	case CPU_R4600:			/* QED style two way caches? */
1116	case CPU_R4700:
1117	case CPU_R5000:
1118	case CPU_NEVADA:
1119		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1120		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1121		c->icache.ways = 2;
1122		c->icache.waybit = __ffs(icache_size/2);
1123
1124		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1125		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1126		c->dcache.ways = 2;
1127		c->dcache.waybit= __ffs(dcache_size/2);
1128
1129		c->options |= MIPS_CPU_CACHE_CDEX_P;
1130		break;
1131
 
1132	case CPU_R5500:
1133		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1134		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1135		c->icache.ways = 2;
1136		c->icache.waybit= 0;
1137
1138		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1139		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1140		c->dcache.ways = 2;
1141		c->dcache.waybit = 0;
1142
1143		c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH;
1144		break;
1145
1146	case CPU_TX49XX:
1147		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1148		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1149		c->icache.ways = 4;
1150		c->icache.waybit= 0;
1151
1152		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1153		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1154		c->dcache.ways = 4;
1155		c->dcache.waybit = 0;
1156
1157		c->options |= MIPS_CPU_CACHE_CDEX_P;
1158		c->options |= MIPS_CPU_PREFETCH;
1159		break;
1160
1161	case CPU_R4000PC:
1162	case CPU_R4000SC:
1163	case CPU_R4000MC:
1164	case CPU_R4400PC:
1165	case CPU_R4400SC:
1166	case CPU_R4400MC:
1167	case CPU_R4300:
1168		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1169		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1170		c->icache.ways = 1;
1171		c->icache.waybit = 0;	/* doesn't matter */
1172
1173		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1174		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1175		c->dcache.ways = 1;
1176		c->dcache.waybit = 0;	/* does not matter */
1177
1178		c->options |= MIPS_CPU_CACHE_CDEX_P;
1179		break;
1180
1181	case CPU_R10000:
1182	case CPU_R12000:
1183	case CPU_R14000:
1184	case CPU_R16000:
1185		icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
1186		c->icache.linesz = 64;
1187		c->icache.ways = 2;
1188		c->icache.waybit = 0;
1189
1190		dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
1191		c->dcache.linesz = 32;
1192		c->dcache.ways = 2;
1193		c->dcache.waybit = 0;
1194
1195		c->options |= MIPS_CPU_PREFETCH;
1196		break;
1197
1198	case CPU_VR4133:
1199		write_c0_config(config & ~VR41_CONF_P4K);
1200		fallthrough;
1201	case CPU_VR4131:
1202		/* Workaround for cache instruction bug of VR4131 */
1203		if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
1204		    c->processor_id == 0x0c82U) {
1205			config |= 0x00400000U;
1206			if (c->processor_id == 0x0c80U)
1207				config |= VR41_CONF_BP;
1208			write_c0_config(config);
1209		} else
1210			c->options |= MIPS_CPU_CACHE_CDEX_P;
1211
1212		icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
1213		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1214		c->icache.ways = 2;
1215		c->icache.waybit = __ffs(icache_size/2);
1216
1217		dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
1218		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1219		c->dcache.ways = 2;
1220		c->dcache.waybit = __ffs(dcache_size/2);
1221		break;
1222
1223	case CPU_VR41XX:
1224	case CPU_VR4111:
1225	case CPU_VR4121:
1226	case CPU_VR4122:
1227	case CPU_VR4181:
1228	case CPU_VR4181A:
1229		icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
1230		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1231		c->icache.ways = 1;
1232		c->icache.waybit = 0;	/* doesn't matter */
1233
1234		dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
1235		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1236		c->dcache.ways = 1;
1237		c->dcache.waybit = 0;	/* does not matter */
1238
1239		c->options |= MIPS_CPU_CACHE_CDEX_P;
1240		break;
1241
1242	case CPU_RM7000:
1243		rm7k_erratum31();
1244
 
1245		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1246		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1247		c->icache.ways = 4;
1248		c->icache.waybit = __ffs(icache_size / c->icache.ways);
1249
1250		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1251		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1252		c->dcache.ways = 4;
1253		c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
1254
 
1255		c->options |= MIPS_CPU_CACHE_CDEX_P;
 
1256		c->options |= MIPS_CPU_PREFETCH;
1257		break;
1258
1259	case CPU_LOONGSON2EF:
1260		icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1261		c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1262		if (prid & 0x3)
1263			c->icache.ways = 4;
1264		else
1265			c->icache.ways = 2;
1266		c->icache.waybit = 0;
1267
1268		dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1269		c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1270		if (prid & 0x3)
1271			c->dcache.ways = 4;
1272		else
1273			c->dcache.ways = 2;
1274		c->dcache.waybit = 0;
1275		break;
1276
1277	case CPU_LOONGSON64:
1278		config1 = read_c0_config1();
1279		lsize = (config1 >> 19) & 7;
1280		if (lsize)
1281			c->icache.linesz = 2 << lsize;
1282		else
1283			c->icache.linesz = 0;
1284		c->icache.sets = 64 << ((config1 >> 22) & 7);
1285		c->icache.ways = 1 + ((config1 >> 16) & 7);
1286		icache_size = c->icache.sets *
1287					  c->icache.ways *
1288					  c->icache.linesz;
1289		c->icache.waybit = 0;
1290
1291		lsize = (config1 >> 10) & 7;
1292		if (lsize)
1293			c->dcache.linesz = 2 << lsize;
1294		else
1295			c->dcache.linesz = 0;
1296		c->dcache.sets = 64 << ((config1 >> 13) & 7);
1297		c->dcache.ways = 1 + ((config1 >> 7) & 7);
1298		dcache_size = c->dcache.sets *
1299					  c->dcache.ways *
1300					  c->dcache.linesz;
1301		c->dcache.waybit = 0;
1302		if ((c->processor_id & (PRID_IMP_MASK | PRID_REV_MASK)) >=
1303				(PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0) ||
1304				(c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R)
1305			c->options |= MIPS_CPU_PREFETCH;
1306		break;
1307
1308	case CPU_CAVIUM_OCTEON3:
1309		/* For now lie about the number of ways. */
1310		c->icache.linesz = 128;
1311		c->icache.sets = 16;
1312		c->icache.ways = 8;
1313		c->icache.flags |= MIPS_CACHE_VTAG;
1314		icache_size = c->icache.sets * c->icache.ways * c->icache.linesz;
1315
1316		c->dcache.linesz = 128;
1317		c->dcache.ways = 8;
1318		c->dcache.sets = 8;
1319		dcache_size = c->dcache.sets * c->dcache.ways * c->dcache.linesz;
1320		c->options |= MIPS_CPU_PREFETCH;
1321		break;
1322
1323	default:
1324		if (!(config & MIPS_CONF_M))
1325			panic("Don't know how to probe P-caches on this cpu.");
1326
1327		/*
1328		 * So we seem to be a MIPS32 or MIPS64 CPU
1329		 * So let's probe the I-cache ...
1330		 */
1331		config1 = read_c0_config1();
1332
1333		lsize = (config1 >> 19) & 7;
1334
1335		/* IL == 7 is reserved */
1336		if (lsize == 7)
1337			panic("Invalid icache line size");
1338
1339		c->icache.linesz = lsize ? 2 << lsize : 0;
1340
1341		c->icache.sets = 32 << (((config1 >> 22) + 1) & 7);
1342		c->icache.ways = 1 + ((config1 >> 16) & 7);
1343
1344		icache_size = c->icache.sets *
1345			      c->icache.ways *
1346			      c->icache.linesz;
1347		c->icache.waybit = __ffs(icache_size/c->icache.ways);
1348
1349		if (config & MIPS_CONF_VI)
1350			c->icache.flags |= MIPS_CACHE_VTAG;
1351
1352		/*
1353		 * Now probe the MIPS32 / MIPS64 data cache.
1354		 */
1355		c->dcache.flags = 0;
1356
1357		lsize = (config1 >> 10) & 7;
1358
1359		/* DL == 7 is reserved */
1360		if (lsize == 7)
1361			panic("Invalid dcache line size");
1362
1363		c->dcache.linesz = lsize ? 2 << lsize : 0;
1364
1365		c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7);
1366		c->dcache.ways = 1 + ((config1 >> 7) & 7);
1367
1368		dcache_size = c->dcache.sets *
1369			      c->dcache.ways *
1370			      c->dcache.linesz;
1371		c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
1372
1373		c->options |= MIPS_CPU_PREFETCH;
1374		break;
1375	}
1376
1377	/*
1378	 * Processor configuration sanity check for the R4000SC erratum
1379	 * #5.	With page sizes larger than 32kB there is no possibility
1380	 * to get a VCE exception anymore so we don't care about this
1381	 * misconfiguration.  The case is rather theoretical anyway;
1382	 * presumably no vendor is shipping his hardware in the "bad"
1383	 * configuration.
1384	 */
1385	if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 &&
1386	    (prid & PRID_REV_MASK) < PRID_REV_R4400 &&
1387	    !(config & CONF_SC) && c->icache.linesz != 16 &&
1388	    PAGE_SIZE <= 0x8000)
1389		panic("Improper R4000SC processor configuration detected");
1390
1391	/* compute a couple of other cache variables */
1392	c->icache.waysize = icache_size / c->icache.ways;
1393	c->dcache.waysize = dcache_size / c->dcache.ways;
1394
1395	c->icache.sets = c->icache.linesz ?
1396		icache_size / (c->icache.linesz * c->icache.ways) : 0;
1397	c->dcache.sets = c->dcache.linesz ?
1398		dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
1399
1400	/*
1401	 * R1x000 P-caches are odd in a positive way.  They're 32kB 2-way
1402	 * virtually indexed so normally would suffer from aliases.  So
1403	 * normally they'd suffer from aliases but magic in the hardware deals
1404	 * with that for us so we don't need to take care ourselves.
1405	 */
1406	switch (current_cpu_type()) {
1407	case CPU_20KC:
1408	case CPU_25KF:
1409	case CPU_I6400:
1410	case CPU_I6500:
1411	case CPU_SB1:
1412	case CPU_SB1A:
1413	case CPU_XLR:
1414		c->dcache.flags |= MIPS_CACHE_PINDEX;
1415		break;
1416
1417	case CPU_R10000:
1418	case CPU_R12000:
1419	case CPU_R14000:
1420	case CPU_R16000:
1421		break;
1422
1423	case CPU_74K:
1424	case CPU_1074K:
1425		has_74k_erratum = alias_74k_erratum(c);
1426		fallthrough;
1427	case CPU_M14KC:
1428	case CPU_M14KEC:
1429	case CPU_24K:
1430	case CPU_34K:
 
1431	case CPU_1004K:
1432	case CPU_INTERAPTIV:
1433	case CPU_P5600:
1434	case CPU_PROAPTIV:
1435	case CPU_M5150:
1436	case CPU_QEMU_GENERIC:
1437	case CPU_P6600:
1438	case CPU_M6250:
1439		if (!(read_c0_config7() & MIPS_CONF7_IAR) &&
1440		    (c->icache.waysize > PAGE_SIZE))
1441			c->icache.flags |= MIPS_CACHE_ALIASES;
1442		if (!has_74k_erratum && (read_c0_config7() & MIPS_CONF7_AR)) {
1443			/*
1444			 * Effectively physically indexed dcache,
1445			 * thus no virtual aliases.
1446			*/
1447			c->dcache.flags |= MIPS_CACHE_PINDEX;
1448			break;
1449		}
1450		fallthrough;
1451	default:
1452		if (has_74k_erratum || c->dcache.waysize > PAGE_SIZE)
1453			c->dcache.flags |= MIPS_CACHE_ALIASES;
1454	}
1455
1456	/* Physically indexed caches don't suffer from virtual aliasing */
1457	if (c->dcache.flags & MIPS_CACHE_PINDEX)
1458		c->dcache.flags &= ~MIPS_CACHE_ALIASES;
1459
1460	/*
1461	 * In systems with CM the icache fills from L2 or closer caches, and
1462	 * thus sees remote stores without needing to write them back any
1463	 * further than that.
1464	 */
1465	if (mips_cm_present())
1466		c->icache.flags |= MIPS_IC_SNOOPS_REMOTE;
1467
1468	switch (current_cpu_type()) {
1469	case CPU_20KC:
1470		/*
1471		 * Some older 20Kc chips doesn't have the 'VI' bit in
1472		 * the config register.
1473		 */
1474		c->icache.flags |= MIPS_CACHE_VTAG;
1475		break;
1476
1477	case CPU_ALCHEMY:
1478	case CPU_I6400:
1479	case CPU_I6500:
1480		c->icache.flags |= MIPS_CACHE_IC_F_DC;
1481		break;
1482
1483	case CPU_BMIPS5000:
1484		c->icache.flags |= MIPS_CACHE_IC_F_DC;
1485		/* Cache aliases are handled in hardware; allow HIGHMEM */
1486		c->dcache.flags &= ~MIPS_CACHE_ALIASES;
1487		break;
1488
1489	case CPU_LOONGSON2EF:
1490		/*
1491		 * LOONGSON2 has 4 way icache, but when using indexed cache op,
1492		 * one op will act on all 4 ways
1493		 */
1494		c->icache.ways = 1;
1495	}
1496
1497	pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1498		icache_size >> 10,
1499		c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT",
1500		way_string[c->icache.ways], c->icache.linesz);
1501
1502	pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
1503		dcache_size >> 10, way_string[c->dcache.ways],
1504		(c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT",
1505		(c->dcache.flags & MIPS_CACHE_ALIASES) ?
1506			"cache aliases" : "no aliases",
1507		c->dcache.linesz);
1508}
1509
1510static void probe_vcache(void)
1511{
1512	struct cpuinfo_mips *c = &current_cpu_data;
1513	unsigned int config2, lsize;
1514
1515	if (current_cpu_type() != CPU_LOONGSON64)
1516		return;
1517
1518	config2 = read_c0_config2();
1519	if ((lsize = ((config2 >> 20) & 15)))
1520		c->vcache.linesz = 2 << lsize;
1521	else
1522		c->vcache.linesz = lsize;
1523
1524	c->vcache.sets = 64 << ((config2 >> 24) & 15);
1525	c->vcache.ways = 1 + ((config2 >> 16) & 15);
1526
1527	vcache_size = c->vcache.sets * c->vcache.ways * c->vcache.linesz;
1528
1529	c->vcache.waybit = 0;
1530	c->vcache.waysize = vcache_size / c->vcache.ways;
1531
1532	pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n",
1533		vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz);
 
 
 
 
 
 
 
 
 
1534}
1535
1536/*
1537 * If you even _breathe_ on this function, look at the gcc output and make sure
1538 * it does not pop things on and off the stack for the cache sizing loop that
1539 * executes in KSEG1 space or else you will crash and burn badly.  You have
1540 * been warned.
1541 */
1542static int probe_scache(void)
1543{
1544	unsigned long flags, addr, begin, end, pow2;
1545	unsigned int config = read_c0_config();
1546	struct cpuinfo_mips *c = &current_cpu_data;
1547
1548	if (config & CONF_SC)
1549		return 0;
1550
1551	begin = (unsigned long) &_stext;
1552	begin &= ~((4 * 1024 * 1024) - 1);
1553	end = begin + (4 * 1024 * 1024);
1554
1555	/*
1556	 * This is such a bitch, you'd think they would make it easy to do
1557	 * this.  Away you daemons of stupidity!
1558	 */
1559	local_irq_save(flags);
1560
1561	/* Fill each size-multiple cache line with a valid tag. */
1562	pow2 = (64 * 1024);
1563	for (addr = begin; addr < end; addr = (begin + pow2)) {
1564		unsigned long *p = (unsigned long *) addr;
1565		__asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1566		pow2 <<= 1;
1567	}
1568
1569	/* Load first line with zero (therefore invalid) tag. */
1570	write_c0_taglo(0);
1571	write_c0_taghi(0);
1572	__asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1573	cache_op(Index_Store_Tag_I, begin);
1574	cache_op(Index_Store_Tag_D, begin);
1575	cache_op(Index_Store_Tag_SD, begin);
1576
1577	/* Now search for the wrap around point. */
1578	pow2 = (128 * 1024);
1579	for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1580		cache_op(Index_Load_Tag_SD, addr);
1581		__asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1582		if (!read_c0_taglo())
1583			break;
1584		pow2 <<= 1;
1585	}
1586	local_irq_restore(flags);
1587	addr -= begin;
1588
1589	scache_size = addr;
1590	c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1591	c->scache.ways = 1;
1592	c->scache.waybit = 0;		/* does not matter */
1593
1594	return 1;
1595}
1596
1597static void loongson2_sc_init(void)
 
1598{
1599	struct cpuinfo_mips *c = &current_cpu_data;
1600
1601	scache_size = 512*1024;
1602	c->scache.linesz = 32;
1603	c->scache.ways = 4;
1604	c->scache.waybit = 0;
1605	c->scache.waysize = scache_size / (c->scache.ways);
1606	c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1607	pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1608	       scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1609
1610	c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1611}
1612
1613static void loongson3_sc_init(void)
1614{
1615	struct cpuinfo_mips *c = &current_cpu_data;
1616	unsigned int config2, lsize;
1617
1618	config2 = read_c0_config2();
1619	lsize = (config2 >> 4) & 15;
1620	if (lsize)
1621		c->scache.linesz = 2 << lsize;
1622	else
1623		c->scache.linesz = 0;
1624	c->scache.sets = 64 << ((config2 >> 8) & 15);
1625	c->scache.ways = 1 + (config2 & 15);
1626
1627	/* Loongson-3 has 4-Scache banks, while Loongson-2K have only 2 banks */
1628	if ((c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R)
1629		c->scache.sets *= 2;
1630	else
1631		c->scache.sets *= 4;
1632
1633	scache_size = c->scache.sets * c->scache.ways * c->scache.linesz;
1634
1635	c->scache.waybit = 0;
1636	c->scache.waysize = scache_size / c->scache.ways;
1637	pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1638	       scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1639	if (scache_size)
1640		c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1641	return;
1642}
1643
1644extern int r5k_sc_init(void);
1645extern int rm7k_sc_init(void);
1646extern int mips_sc_init(void);
1647
1648static void setup_scache(void)
1649{
1650	struct cpuinfo_mips *c = &current_cpu_data;
1651	unsigned int config = read_c0_config();
1652	int sc_present = 0;
1653
1654	/*
1655	 * Do the probing thing on R4000SC and R4400SC processors.  Other
1656	 * processors don't have a S-cache that would be relevant to the
1657	 * Linux memory management.
1658	 */
1659	switch (current_cpu_type()) {
1660	case CPU_R4000SC:
1661	case CPU_R4000MC:
1662	case CPU_R4400SC:
1663	case CPU_R4400MC:
1664		sc_present = run_uncached(probe_scache);
1665		if (sc_present)
1666			c->options |= MIPS_CPU_CACHE_CDEX_S;
1667		break;
1668
1669	case CPU_R10000:
1670	case CPU_R12000:
1671	case CPU_R14000:
1672	case CPU_R16000:
1673		scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1674		c->scache.linesz = 64 << ((config >> 13) & 1);
1675		c->scache.ways = 2;
1676		c->scache.waybit= 0;
1677		sc_present = 1;
1678		break;
1679
1680	case CPU_R5000:
1681	case CPU_NEVADA:
1682#ifdef CONFIG_R5000_CPU_SCACHE
1683		r5k_sc_init();
1684#endif
1685		return;
1686
1687	case CPU_RM7000:
 
1688#ifdef CONFIG_RM7000_CPU_SCACHE
1689		rm7k_sc_init();
1690#endif
1691		return;
1692
1693	case CPU_LOONGSON2EF:
 
1694		loongson2_sc_init();
1695		return;
1696
1697	case CPU_LOONGSON64:
1698		loongson3_sc_init();
1699		return;
1700
1701	case CPU_CAVIUM_OCTEON3:
1702	case CPU_XLP:
1703		/* don't need to worry about L2, fully coherent */
1704		return;
1705
1706	default:
1707		if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 |
1708				    MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 |
1709				    MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 |
1710				    MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) {
1711#ifdef CONFIG_MIPS_CPU_SCACHE
1712			if (mips_sc_init ()) {
1713				scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
1714				printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
1715				       scache_size >> 10,
1716				       way_string[c->scache.ways], c->scache.linesz);
1717
1718				if (current_cpu_type() == CPU_BMIPS5000)
1719					c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1720			}
1721
1722#else
1723			if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1724				panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1725#endif
1726			return;
1727		}
1728		sc_present = 0;
1729	}
1730
1731	if (!sc_present)
1732		return;
1733
1734	/* compute a couple of other cache variables */
1735	c->scache.waysize = scache_size / c->scache.ways;
1736
1737	c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1738
1739	printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1740	       scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1741
1742	c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1743}
1744
1745void au1x00_fixup_config_od(void)
1746{
1747	/*
1748	 * c0_config.od (bit 19) was write only (and read as 0)
1749	 * on the early revisions of Alchemy SOCs.  It disables the bus
1750	 * transaction overlapping and needs to be set to fix various errata.
1751	 */
1752	switch (read_c0_prid()) {
1753	case 0x00030100: /* Au1000 DA */
1754	case 0x00030201: /* Au1000 HA */
1755	case 0x00030202: /* Au1000 HB */
1756	case 0x01030200: /* Au1500 AB */
1757	/*
1758	 * Au1100 errata actually keeps silence about this bit, so we set it
1759	 * just in case for those revisions that require it to be set according
1760	 * to the (now gone) cpu table.
1761	 */
1762	case 0x02030200: /* Au1100 AB */
1763	case 0x02030201: /* Au1100 BA */
1764	case 0x02030202: /* Au1100 BC */
1765		set_c0_config(1 << 19);
1766		break;
1767	}
1768}
1769
1770/* CP0 hazard avoidance. */
1771#define NXP_BARRIER()							\
1772	 __asm__ __volatile__(						\
1773	".set noreorder\n\t"						\
1774	"nop; nop; nop; nop; nop; nop;\n\t"				\
1775	".set reorder\n\t")
1776
1777static void nxp_pr4450_fixup_config(void)
1778{
1779	unsigned long config0;
1780
1781	config0 = read_c0_config();
1782
1783	/* clear all three cache coherency fields */
1784	config0 &= ~(0x7 | (7 << 25) | (7 << 28));
1785	config0 |= (((_page_cachable_default >> _CACHE_SHIFT) <<  0) |
1786		    ((_page_cachable_default >> _CACHE_SHIFT) << 25) |
1787		    ((_page_cachable_default >> _CACHE_SHIFT) << 28));
1788	write_c0_config(config0);
1789	NXP_BARRIER();
1790}
1791
1792static int cca = -1;
1793
1794static int __init cca_setup(char *str)
1795{
1796	get_option(&str, &cca);
1797
1798	return 0;
1799}
1800
1801early_param("cca", cca_setup);
1802
1803static void coherency_setup(void)
1804{
1805	if (cca < 0 || cca > 7)
1806		cca = read_c0_config() & CONF_CM_CMASK;
1807	_page_cachable_default = cca << _CACHE_SHIFT;
1808
1809	pr_debug("Using cache attribute %d\n", cca);
1810	change_c0_config(CONF_CM_CMASK, cca);
1811
1812	/*
1813	 * c0_status.cu=0 specifies that updates by the sc instruction use
1814	 * the coherency mode specified by the TLB; 1 means cachable
1815	 * coherent update on write will be used.  Not all processors have
1816	 * this bit and; some wire it to zero, others like Toshiba had the
1817	 * silly idea of putting something else there ...
1818	 */
1819	switch (current_cpu_type()) {
1820	case CPU_R4000PC:
1821	case CPU_R4000SC:
1822	case CPU_R4000MC:
1823	case CPU_R4400PC:
1824	case CPU_R4400SC:
1825	case CPU_R4400MC:
1826		clear_c0_config(CONF_CU);
1827		break;
1828	/*
1829	 * We need to catch the early Alchemy SOCs with
1830	 * the write-only co_config.od bit and set it back to one on:
1831	 * Au1000 rev DA, HA, HB;  Au1100 AB, BA, BC, Au1500 AB
1832	 */
1833	case CPU_ALCHEMY:
1834		au1x00_fixup_config_od();
1835		break;
1836
1837	case PRID_IMP_PR4450:
1838		nxp_pr4450_fixup_config();
1839		break;
1840	}
1841}
1842
1843static void r4k_cache_error_setup(void)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1844{
1845	extern char __weak except_vec2_generic;
1846	extern char __weak except_vec2_sb1;
 
1847
1848	switch (current_cpu_type()) {
1849	case CPU_SB1:
1850	case CPU_SB1A:
1851		set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
1852		break;
1853
1854	default:
1855		set_uncached_handler(0x100, &except_vec2_generic, 0x80);
1856		break;
1857	}
1858}
1859
1860void r4k_cache_init(void)
1861{
1862	extern void build_clear_page(void);
1863	extern void build_copy_page(void);
1864	struct cpuinfo_mips *c = &current_cpu_data;
1865
1866	probe_pcache();
1867	probe_vcache();
1868	setup_scache();
1869
1870	r4k_blast_dcache_page_setup();
1871	r4k_blast_dcache_page_indexed_setup();
1872	r4k_blast_dcache_setup();
1873	r4k_blast_icache_page_setup();
1874	r4k_blast_icache_page_indexed_setup();
1875	r4k_blast_icache_setup();
1876	r4k_blast_scache_page_setup();
1877	r4k_blast_scache_page_indexed_setup();
1878	r4k_blast_scache_setup();
1879	r4k_blast_scache_node_setup();
1880#ifdef CONFIG_EVA
1881	r4k_blast_dcache_user_page_setup();
1882	r4k_blast_icache_user_page_setup();
1883#endif
1884
1885	/*
1886	 * Some MIPS32 and MIPS64 processors have physically indexed caches.
1887	 * This code supports virtually indexed processors and will be
1888	 * unnecessarily inefficient on physically indexed processors.
1889	 */
1890	if (c->dcache.linesz && cpu_has_dc_aliases)
1891		shm_align_mask = max_t( unsigned long,
1892					c->dcache.sets * c->dcache.linesz - 1,
1893					PAGE_SIZE - 1);
1894	else
1895		shm_align_mask = PAGE_SIZE-1;
1896
1897	__flush_cache_vmap	= r4k__flush_cache_vmap;
1898	__flush_cache_vunmap	= r4k__flush_cache_vunmap;
1899
1900	flush_cache_all		= cache_noop;
1901	__flush_cache_all	= r4k___flush_cache_all;
1902	flush_cache_mm		= r4k_flush_cache_mm;
1903	flush_cache_page	= r4k_flush_cache_page;
1904	flush_cache_range	= r4k_flush_cache_range;
1905
1906	__flush_kernel_vmap_range = r4k_flush_kernel_vmap_range;
1907
 
1908	flush_icache_all	= r4k_flush_icache_all;
1909	local_flush_data_cache_page	= local_r4k_flush_data_cache_page;
1910	flush_data_cache_page	= r4k_flush_data_cache_page;
1911	flush_icache_range	= r4k_flush_icache_range;
1912	local_flush_icache_range	= local_r4k_flush_icache_range;
1913	__flush_icache_user_range	= r4k_flush_icache_user_range;
1914	__local_flush_icache_user_range	= local_r4k_flush_icache_user_range;
1915
1916#ifdef CONFIG_DMA_NONCOHERENT
1917	if (dma_default_coherent) {
1918		_dma_cache_wback_inv	= (void *)cache_noop;
1919		_dma_cache_wback	= (void *)cache_noop;
1920		_dma_cache_inv		= (void *)cache_noop;
1921	} else {
1922		_dma_cache_wback_inv	= r4k_dma_cache_wback_inv;
1923		_dma_cache_wback	= r4k_dma_cache_wback_inv;
1924		_dma_cache_inv		= r4k_dma_cache_inv;
1925	}
1926#endif /* CONFIG_DMA_NONCOHERENT */
1927
1928	build_clear_page();
1929	build_copy_page();
1930
1931	/*
1932	 * We want to run CMP kernels on core with and without coherent
1933	 * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether
1934	 * or not to flush caches.
1935	 */
1936	local_r4k___flush_cache_all(NULL);
1937
1938	coherency_setup();
1939	board_cache_error_setup = r4k_cache_error_setup;
1940
1941	/*
1942	 * Per-CPU overrides
1943	 */
1944	switch (current_cpu_type()) {
1945	case CPU_BMIPS4350:
1946	case CPU_BMIPS4380:
1947		/* No IPI is needed because all CPUs share the same D$ */
1948		flush_data_cache_page = r4k_blast_dcache_page;
1949		break;
1950	case CPU_BMIPS5000:
1951		/* We lose our superpowers if L2 is disabled */
1952		if (c->scache.flags & MIPS_CACHE_NOT_PRESENT)
1953			break;
1954
1955		/* I$ fills from D$ just by emptying the write buffers */
1956		flush_cache_page = (void *)b5k_instruction_hazard;
1957		flush_cache_range = (void *)b5k_instruction_hazard;
1958		local_flush_data_cache_page = (void *)b5k_instruction_hazard;
1959		flush_data_cache_page = (void *)b5k_instruction_hazard;
1960		flush_icache_range = (void *)b5k_instruction_hazard;
1961		local_flush_icache_range = (void *)b5k_instruction_hazard;
1962
1963
1964		/* Optimization: an L2 flush implicitly flushes the L1 */
1965		current_cpu_data.options |= MIPS_CPU_INCLUSIVE_CACHES;
1966		break;
1967	case CPU_LOONGSON64:
1968		/* Loongson-3 maintains cache coherency by hardware */
1969		__flush_cache_all	= cache_noop;
1970		__flush_cache_vmap	= cache_noop;
1971		__flush_cache_vunmap	= cache_noop;
1972		__flush_kernel_vmap_range = (void *)cache_noop;
1973		flush_cache_mm		= (void *)cache_noop;
1974		flush_cache_page	= (void *)cache_noop;
1975		flush_cache_range	= (void *)cache_noop;
1976		flush_icache_all	= (void *)cache_noop;
1977		flush_data_cache_page	= (void *)cache_noop;
1978		local_flush_data_cache_page	= (void *)cache_noop;
1979		break;
1980	}
1981}
1982
1983static int r4k_cache_pm_notifier(struct notifier_block *self, unsigned long cmd,
1984			       void *v)
1985{
1986	switch (cmd) {
1987	case CPU_PM_ENTER_FAILED:
1988	case CPU_PM_EXIT:
1989		coherency_setup();
1990		break;
1991	}
1992
1993	return NOTIFY_OK;
1994}
1995
1996static struct notifier_block r4k_cache_pm_notifier_block = {
1997	.notifier_call = r4k_cache_pm_notifier,
1998};
1999
2000int __init r4k_cache_init_pm(void)
2001{
2002	return cpu_pm_register_notifier(&r4k_cache_pm_notifier_block);
2003}
2004arch_initcall(r4k_cache_init_pm);