blob: c14259edd53f40539ce41cf73025ddce6cbcb277 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
Justin P. Mattock79add622011-04-04 14:15:29 -07006 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
9 */
Ralf Baechlea754f702007-11-03 01:01:37 +000010#include <linux/hardirq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/init.h>
Ralf Baechledb813fe2007-09-27 18:26:43 +010012#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/kernel.h>
Ralf Baechle641e97f2007-10-11 23:46:05 +010014#include <linux/linkage.h>
Ralf Baechleff522052013-09-17 12:44:31 +020015#include <linux/preempt.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/sched.h>
Ralf Baechle631330f2009-06-19 14:05:26 +010017#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/mm.h>
Chris Dearman35133692007-09-19 00:58:24 +010019#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/bitops.h>
21
22#include <asm/bcache.h>
23#include <asm/bootinfo.h>
Ralf Baechleec74e362005-07-13 11:48:45 +000024#include <asm/cache.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <asm/cacheops.h>
26#include <asm/cpu.h>
27#include <asm/cpu-features.h>
Ralf Baechle69f24d12013-09-17 10:25:47 +020028#include <asm/cpu-type.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <asm/io.h>
30#include <asm/page.h>
31#include <asm/pgtable.h>
32#include <asm/r4kcache.h>
Ralf Baechlee001e522007-07-28 12:45:47 +010033#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <asm/mmu_context.h>
35#include <asm/war.h>
Thiemo Seuferba5187d2005-04-25 16:36:23 +000036#include <asm/cacheflush.h> /* for run_uncached() */
David Daney9cd9669b2012-05-15 00:04:49 -070037#include <asm/traps.h>
Steven J. Hillb6d92b42013-03-25 13:47:29 -050038#include <asm/dma-coherence.h>
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010039
40/*
41 * Special Variant of smp_call_function for use by cache functions:
42 *
43 * o No return value
44 * o collapses to normal function call on UP kernels
45 * o collapses to normal function call on systems with a single shared
46 * primary cache.
Ralf Baechlec8c5f3f2010-10-29 19:08:25 +010047 * o doesn't disable interrupts on the local CPU
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010048 */
Ralf Baechle48a26e62010-10-29 19:08:25 +010049static inline void r4k_on_each_cpu(void (*func) (void *info), void *info)
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010050{
51 preempt_disable();
52
53#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
Ralf Baechle48a26e62010-10-29 19:08:25 +010054 smp_call_function(func, info, 1);
Ralf Baechle7f3f1d02006-05-12 13:20:06 +010055#endif
56 func(info);
57 preempt_enable();
58}
59
Ralf Baechle39b8d522008-04-28 17:14:26 +010060#if defined(CONFIG_MIPS_CMP)
61#define cpu_has_safe_index_cacheops 0
62#else
63#define cpu_has_safe_index_cacheops 1
64#endif
65
Ralf Baechleec74e362005-07-13 11:48:45 +000066/*
67 * Must die.
68 */
69static unsigned long icache_size __read_mostly;
70static unsigned long dcache_size __read_mostly;
71static unsigned long scache_size __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070072
73/*
74 * Dummy cache handling routines for machines without boardcaches
75 */
Chris Dearman73f40352006-06-20 18:06:52 +010076static void cache_noop(void) {}
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
78static struct bcache_ops no_sc_ops = {
Chris Dearman73f40352006-06-20 18:06:52 +010079 .bc_enable = (void *)cache_noop,
80 .bc_disable = (void *)cache_noop,
81 .bc_wback_inv = (void *)cache_noop,
82 .bc_inv = (void *)cache_noop
Linus Torvalds1da177e2005-04-16 15:20:36 -070083};
84
85struct bcache_ops *bcops = &no_sc_ops;
86
Thiemo Seufer330cfe02005-09-01 18:33:58 +000087#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
88#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
Linus Torvalds1da177e2005-04-16 15:20:36 -070089
90#define R4600_HIT_CACHEOP_WAR_IMPL \
91do { \
92 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \
93 *(volatile unsigned long *)CKSEG1; \
94 if (R4600_V1_HIT_CACHEOP_WAR) \
95 __asm__ __volatile__("nop;nop;nop;nop"); \
96} while (0)
97
98static void (*r4k_blast_dcache_page)(unsigned long addr);
99
100static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
101{
102 R4600_HIT_CACHEOP_WAR_IMPL;
103 blast_dcache32_page(addr);
104}
105
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700106static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
107{
108 R4600_HIT_CACHEOP_WAR_IMPL;
109 blast_dcache64_page(addr);
110}
111
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000112static void r4k_blast_dcache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113{
114 unsigned long dc_lsize = cpu_dcache_line_size();
115
Chris Dearman73f40352006-06-20 18:06:52 +0100116 if (dc_lsize == 0)
117 r4k_blast_dcache_page = (void *)cache_noop;
118 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119 r4k_blast_dcache_page = blast_dcache16_page;
120 else if (dc_lsize == 32)
121 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700122 else if (dc_lsize == 64)
123 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124}
125
126static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
127
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000128static void r4k_blast_dcache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129{
130 unsigned long dc_lsize = cpu_dcache_line_size();
131
Chris Dearman73f40352006-06-20 18:06:52 +0100132 if (dc_lsize == 0)
133 r4k_blast_dcache_page_indexed = (void *)cache_noop;
134 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
136 else if (dc_lsize == 32)
137 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700138 else if (dc_lsize == 64)
139 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140}
141
Sanjay Lalf2e36562012-11-21 18:34:10 -0800142void (* r4k_blast_dcache)(void);
143EXPORT_SYMBOL(r4k_blast_dcache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000145static void r4k_blast_dcache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146{
147 unsigned long dc_lsize = cpu_dcache_line_size();
148
Chris Dearman73f40352006-06-20 18:06:52 +0100149 if (dc_lsize == 0)
150 r4k_blast_dcache = (void *)cache_noop;
151 else if (dc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 r4k_blast_dcache = blast_dcache16;
153 else if (dc_lsize == 32)
154 r4k_blast_dcache = blast_dcache32;
Kevin Cernekee605b7ef2009-04-23 17:36:53 -0700155 else if (dc_lsize == 64)
156 r4k_blast_dcache = blast_dcache64;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157}
158
159/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
160#define JUMP_TO_ALIGN(order) \
161 __asm__ __volatile__( \
162 "b\t1f\n\t" \
163 ".align\t" #order "\n\t" \
164 "1:\n\t" \
165 )
166#define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
Ralf Baechle70342282013-01-22 12:59:30 +0100167#define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
169static inline void blast_r4600_v1_icache32(void)
170{
171 unsigned long flags;
172
173 local_irq_save(flags);
174 blast_icache32();
175 local_irq_restore(flags);
176}
177
178static inline void tx49_blast_icache32(void)
179{
180 unsigned long start = INDEX_BASE;
181 unsigned long end = start + current_cpu_data.icache.waysize;
182 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
183 unsigned long ws_end = current_cpu_data.icache.ways <<
Ralf Baechle70342282013-01-22 12:59:30 +0100184 current_cpu_data.icache.waybit;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 unsigned long ws, addr;
186
187 CACHE32_UNROLL32_ALIGN2;
188 /* I'm in even chunk. blast odd chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700189 for (ws = 0; ws < ws_end; ws += ws_inc)
190 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100191 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192 CACHE32_UNROLL32_ALIGN;
193 /* I'm in odd chunk. blast even chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700194 for (ws = 0; ws < ws_end; ws += ws_inc)
195 for (addr = start; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100196 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197}
198
199static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
200{
201 unsigned long flags;
202
203 local_irq_save(flags);
204 blast_icache32_page_indexed(page);
205 local_irq_restore(flags);
206}
207
208static inline void tx49_blast_icache32_page_indexed(unsigned long page)
209{
Atsushi Nemoto67a3f6de2006-04-04 17:34:14 +0900210 unsigned long indexmask = current_cpu_data.icache.waysize - 1;
211 unsigned long start = INDEX_BASE + (page & indexmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 unsigned long end = start + PAGE_SIZE;
213 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
214 unsigned long ws_end = current_cpu_data.icache.ways <<
Ralf Baechle70342282013-01-22 12:59:30 +0100215 current_cpu_data.icache.waybit;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 unsigned long ws, addr;
217
218 CACHE32_UNROLL32_ALIGN2;
219 /* I'm in even chunk. blast odd chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700220 for (ws = 0; ws < ws_end; ws += ws_inc)
221 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100222 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 CACHE32_UNROLL32_ALIGN;
224 /* I'm in odd chunk. blast even chunks */
Ralf Baechle42a3b4f2005-09-03 15:56:17 -0700225 for (ws = 0; ws < ws_end; ws += ws_inc)
226 for (addr = start; addr < end; addr += 0x400 * 2)
Ralf Baechle21a151d2007-10-11 23:46:15 +0100227 cache32_unroll32(addr|ws, Index_Invalidate_I);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228}
229
230static void (* r4k_blast_icache_page)(unsigned long addr);
231
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000232static void r4k_blast_icache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233{
234 unsigned long ic_lsize = cpu_icache_line_size();
235
Chris Dearman73f40352006-06-20 18:06:52 +0100236 if (ic_lsize == 0)
237 r4k_blast_icache_page = (void *)cache_noop;
238 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 r4k_blast_icache_page = blast_icache16_page;
Aaro Koskinen43a06842014-01-14 17:56:38 -0800240 else if (ic_lsize == 32 && current_cpu_type() == CPU_LOONGSON2)
241 r4k_blast_icache_page = loongson2_blast_icache32_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 else if (ic_lsize == 32)
243 r4k_blast_icache_page = blast_icache32_page;
244 else if (ic_lsize == 64)
245 r4k_blast_icache_page = blast_icache64_page;
246}
247
248
249static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
250
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000251static void r4k_blast_icache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252{
253 unsigned long ic_lsize = cpu_icache_line_size();
254
Chris Dearman73f40352006-06-20 18:06:52 +0100255 if (ic_lsize == 0)
256 r4k_blast_icache_page_indexed = (void *)cache_noop;
257 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
259 else if (ic_lsize == 32) {
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000260 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 r4k_blast_icache_page_indexed =
262 blast_icache32_r4600_v1_page_indexed;
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000263 else if (TX49XX_ICACHE_INDEX_INV_WAR)
264 r4k_blast_icache_page_indexed =
265 tx49_blast_icache32_page_indexed;
Aaro Koskinen43a06842014-01-14 17:56:38 -0800266 else if (current_cpu_type() == CPU_LOONGSON2)
267 r4k_blast_icache_page_indexed =
268 loongson2_blast_icache32_page_indexed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 else
270 r4k_blast_icache_page_indexed =
271 blast_icache32_page_indexed;
272 } else if (ic_lsize == 64)
273 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
274}
275
Sanjay Lalf2e36562012-11-21 18:34:10 -0800276void (* r4k_blast_icache)(void);
277EXPORT_SYMBOL(r4k_blast_icache);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000279static void r4k_blast_icache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280{
281 unsigned long ic_lsize = cpu_icache_line_size();
282
Chris Dearman73f40352006-06-20 18:06:52 +0100283 if (ic_lsize == 0)
284 r4k_blast_icache = (void *)cache_noop;
285 else if (ic_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 r4k_blast_icache = blast_icache16;
287 else if (ic_lsize == 32) {
288 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
289 r4k_blast_icache = blast_r4600_v1_icache32;
290 else if (TX49XX_ICACHE_INDEX_INV_WAR)
291 r4k_blast_icache = tx49_blast_icache32;
Aaro Koskinen43a06842014-01-14 17:56:38 -0800292 else if (current_cpu_type() == CPU_LOONGSON2)
293 r4k_blast_icache = loongson2_blast_icache32;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 else
295 r4k_blast_icache = blast_icache32;
296 } else if (ic_lsize == 64)
297 r4k_blast_icache = blast_icache64;
298}
299
300static void (* r4k_blast_scache_page)(unsigned long addr);
301
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000302static void r4k_blast_scache_page_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303{
304 unsigned long sc_lsize = cpu_scache_line_size();
305
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000306 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100307 r4k_blast_scache_page = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000308 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 r4k_blast_scache_page = blast_scache16_page;
310 else if (sc_lsize == 32)
311 r4k_blast_scache_page = blast_scache32_page;
312 else if (sc_lsize == 64)
313 r4k_blast_scache_page = blast_scache64_page;
314 else if (sc_lsize == 128)
315 r4k_blast_scache_page = blast_scache128_page;
316}
317
318static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
319
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000320static void r4k_blast_scache_page_indexed_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321{
322 unsigned long sc_lsize = cpu_scache_line_size();
323
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000324 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100325 r4k_blast_scache_page_indexed = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000326 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
328 else if (sc_lsize == 32)
329 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
330 else if (sc_lsize == 64)
331 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
332 else if (sc_lsize == 128)
333 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
334}
335
336static void (* r4k_blast_scache)(void);
337
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000338static void r4k_blast_scache_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339{
340 unsigned long sc_lsize = cpu_scache_line_size();
341
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000342 if (scache_size == 0)
Chris Dearman73f40352006-06-20 18:06:52 +0100343 r4k_blast_scache = (void *)cache_noop;
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000344 else if (sc_lsize == 16)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 r4k_blast_scache = blast_scache16;
346 else if (sc_lsize == 32)
347 r4k_blast_scache = blast_scache32;
348 else if (sc_lsize == 64)
349 r4k_blast_scache = blast_scache64;
350 else if (sc_lsize == 128)
351 r4k_blast_scache = blast_scache128;
352}
353
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354static inline void local_r4k___flush_cache_all(void * args)
355{
Ralf Baechle10cc3522007-10-11 23:46:15 +0100356 switch (current_cpu_type()) {
Ralf Baechle14bd8c02013-09-25 18:21:26 +0200357 case CPU_LOONGSON2:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 case CPU_R4000SC:
359 case CPU_R4000MC:
360 case CPU_R4400SC:
361 case CPU_R4400MC:
362 case CPU_R10000:
363 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400364 case CPU_R14000:
Ralf Baechle14bd8c02013-09-25 18:21:26 +0200365 /*
366 * These caches are inclusive caches, that is, if something
367 * is not cached in the S-cache, we know it also won't be
368 * in one of the primary caches.
369 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 r4k_blast_scache();
Ralf Baechle14bd8c02013-09-25 18:21:26 +0200371 break;
372
373 default:
374 r4k_blast_dcache();
375 r4k_blast_icache();
376 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 }
378}
379
380static void r4k___flush_cache_all(void)
381{
Ralf Baechle48a26e62010-10-29 19:08:25 +0100382 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383}
384
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100385static inline int has_valid_asid(const struct mm_struct *mm)
386{
387#if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC)
388 int i;
389
390 for_each_online_cpu(i)
391 if (cpu_context(i, mm))
392 return 1;
393
394 return 0;
395#else
396 return cpu_context(smp_processor_id(), mm);
397#endif
398}
399
Ralf Baechle9c5a3d72008-04-05 15:13:23 +0100400static void r4k__flush_cache_vmap(void)
401{
402 r4k_blast_dcache();
403}
404
405static void r4k__flush_cache_vunmap(void)
406{
407 r4k_blast_dcache();
408}
409
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410static inline void local_r4k_flush_cache_range(void * args)
411{
412 struct vm_area_struct *vma = args;
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000413 int exec = vma->vm_flags & VM_EXEC;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700414
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100415 if (!(has_valid_asid(vma->vm_mm)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 return;
417
Atsushi Nemoto0550d9d2006-08-22 21:15:47 +0900418 r4k_blast_dcache();
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000419 if (exec)
420 r4k_blast_icache();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421}
422
423static void r4k_flush_cache_range(struct vm_area_struct *vma,
424 unsigned long start, unsigned long end)
425{
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000426 int exec = vma->vm_flags & VM_EXEC;
Atsushi Nemoto0550d9d2006-08-22 21:15:47 +0900427
Ralf Baechle2eaa7ec2008-02-11 14:51:40 +0000428 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
Ralf Baechle48a26e62010-10-29 19:08:25 +0100429 r4k_on_each_cpu(local_r4k_flush_cache_range, vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430}
431
432static inline void local_r4k_flush_cache_mm(void * args)
433{
434 struct mm_struct *mm = args;
435
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100436 if (!has_valid_asid(mm))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 return;
438
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439 /*
440 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we
441 * only flush the primary caches but R10000 and R12000 behave sane ...
Ralf Baechle617667b2006-11-30 01:14:48 +0000442 * R4000SC and R4400SC indexed S-cache ops also invalidate primary
443 * caches, so we can bail out early.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 */
Ralf Baechle10cc3522007-10-11 23:46:15 +0100445 if (current_cpu_type() == CPU_R4000SC ||
446 current_cpu_type() == CPU_R4000MC ||
447 current_cpu_type() == CPU_R4400SC ||
448 current_cpu_type() == CPU_R4400MC) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 r4k_blast_scache();
Ralf Baechle617667b2006-11-30 01:14:48 +0000450 return;
451 }
452
453 r4k_blast_dcache();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454}
455
456static void r4k_flush_cache_mm(struct mm_struct *mm)
457{
458 if (!cpu_has_dc_aliases)
459 return;
460
Ralf Baechle48a26e62010-10-29 19:08:25 +0100461 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700462}
463
464struct flush_cache_page_args {
465 struct vm_area_struct *vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100466 unsigned long addr;
Atsushi Nemotode628932006-03-13 18:23:03 +0900467 unsigned long pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468};
469
470static inline void local_r4k_flush_cache_page(void *args)
471{
472 struct flush_cache_page_args *fcp_args = args;
473 struct vm_area_struct *vma = fcp_args->vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100474 unsigned long addr = fcp_args->addr;
Ralf Baechledb813fe2007-09-27 18:26:43 +0100475 struct page *page = pfn_to_page(fcp_args->pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700476 int exec = vma->vm_flags & VM_EXEC;
477 struct mm_struct *mm = vma->vm_mm;
Ralf Baechlec9c50232008-06-14 22:22:08 +0100478 int map_coherent = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700479 pgd_t *pgdp;
Ralf Baechlec6e8b582005-02-10 12:19:59 +0000480 pud_t *pudp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481 pmd_t *pmdp;
482 pte_t *ptep;
Ralf Baechledb813fe2007-09-27 18:26:43 +0100483 void *vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484
Ralf Baechle79acf832005-02-10 13:54:37 +0000485 /*
486 * If ownes no valid ASID yet, cannot possibly have gotten
487 * this page into the cache.
488 */
Ralf Baechlea76ab5c2007-10-08 16:38:37 +0100489 if (!has_valid_asid(mm))
Ralf Baechle79acf832005-02-10 13:54:37 +0000490 return;
491
Ralf Baechle6ec25802005-10-12 00:02:34 +0100492 addr &= PAGE_MASK;
493 pgdp = pgd_offset(mm, addr);
494 pudp = pud_offset(pgdp, addr);
495 pmdp = pmd_offset(pudp, addr);
496 ptep = pte_offset(pmdp, addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497
498 /*
499 * If the page isn't marked valid, the page cannot possibly be
500 * in the cache.
501 */
Ralf Baechle526af352008-01-29 10:14:55 +0000502 if (!(pte_present(*ptep)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 return;
504
Ralf Baechledb813fe2007-09-27 18:26:43 +0100505 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
506 vaddr = NULL;
507 else {
508 /*
509 * Use kmap_coherent or kmap_atomic to do flushes for
510 * another ASID than the current one.
511 */
Ralf Baechlec9c50232008-06-14 22:22:08 +0100512 map_coherent = (cpu_has_dc_aliases &&
513 page_mapped(page) && !Page_dcache_dirty(page));
514 if (map_coherent)
Ralf Baechledb813fe2007-09-27 18:26:43 +0100515 vaddr = kmap_coherent(page, addr);
516 else
Cong Wang9c020482011-11-25 23:14:15 +0800517 vaddr = kmap_atomic(page);
Ralf Baechledb813fe2007-09-27 18:26:43 +0100518 addr = (unsigned long)vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 }
520
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
Ralf Baechledb813fe2007-09-27 18:26:43 +0100522 r4k_blast_dcache_page(addr);
Ralf Baechle39b8d522008-04-28 17:14:26 +0100523 if (exec && !cpu_icache_snoops_remote_store)
524 r4k_blast_scache_page(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 }
526 if (exec) {
Ralf Baechledb813fe2007-09-27 18:26:43 +0100527 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 int cpu = smp_processor_id();
529
Thiemo Seufer26a51b22005-02-19 13:32:02 +0000530 if (cpu_context(cpu, mm) != 0)
531 drop_mmu_context(mm, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532 } else
Ralf Baechledb813fe2007-09-27 18:26:43 +0100533 r4k_blast_icache_page(addr);
534 }
535
536 if (vaddr) {
Ralf Baechlec9c50232008-06-14 22:22:08 +0100537 if (map_coherent)
Ralf Baechledb813fe2007-09-27 18:26:43 +0100538 kunmap_coherent();
539 else
Cong Wang9c020482011-11-25 23:14:15 +0800540 kunmap_atomic(vaddr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541 }
542}
543
Ralf Baechle6ec25802005-10-12 00:02:34 +0100544static void r4k_flush_cache_page(struct vm_area_struct *vma,
545 unsigned long addr, unsigned long pfn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546{
547 struct flush_cache_page_args args;
548
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 args.vma = vma;
Ralf Baechle6ec25802005-10-12 00:02:34 +0100550 args.addr = addr;
Atsushi Nemotode628932006-03-13 18:23:03 +0900551 args.pfn = pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552
Ralf Baechle48a26e62010-10-29 19:08:25 +0100553 r4k_on_each_cpu(local_r4k_flush_cache_page, &args);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554}
555
556static inline void local_r4k_flush_data_cache_page(void * addr)
557{
558 r4k_blast_dcache_page((unsigned long) addr);
559}
560
561static void r4k_flush_data_cache_page(unsigned long addr)
562{
Ralf Baechlea754f702007-11-03 01:01:37 +0000563 if (in_atomic())
564 local_r4k_flush_data_cache_page((void *)addr);
565 else
Ralf Baechle48a26e62010-10-29 19:08:25 +0100566 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567}
568
569struct flush_icache_range_args {
Atsushi Nemotod4264f12006-01-29 02:27:51 +0900570 unsigned long start;
571 unsigned long end;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572};
573
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +0200574static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700575{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 if (!cpu_has_ic_fills_f_dc) {
Chris Dearman73f40352006-06-20 18:06:52 +0100577 if (end - start >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578 r4k_blast_dcache();
579 } else {
Thiemo Seufer10a3dab2005-09-09 20:26:54 +0000580 R4600_HIT_CACHEOP_WAR_IMPL;
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900581 protected_blast_dcache_range(start, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 }
584
585 if (end - start > icache_size)
586 r4k_blast_icache();
Ralf Baechle14bd8c02013-09-25 18:21:26 +0200587 else {
588 switch (boot_cpu_type()) {
589 case CPU_LOONGSON2:
Huacai Chenbad009f2014-01-14 17:56:37 -0800590 protected_loongson2_blast_icache_range(start, end);
Ralf Baechle14bd8c02013-09-25 18:21:26 +0200591 break;
592
593 default:
Huacai Chenbad009f2014-01-14 17:56:37 -0800594 protected_blast_icache_range(start, end);
Ralf Baechle14bd8c02013-09-25 18:21:26 +0200595 break;
596 }
597 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598}
599
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +0200600static inline void local_r4k_flush_icache_range_ipi(void *args)
601{
602 struct flush_icache_range_args *fir_args = args;
603 unsigned long start = fir_args->start;
604 unsigned long end = fir_args->end;
605
606 local_r4k_flush_icache_range(start, end);
607}
608
Atsushi Nemotod4264f12006-01-29 02:27:51 +0900609static void r4k_flush_icache_range(unsigned long start, unsigned long end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610{
611 struct flush_icache_range_args args;
612
613 args.start = start;
614 args.end = end;
615
Ralf Baechle48a26e62010-10-29 19:08:25 +0100616 r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args);
Ralf Baechlecc61c1f2005-07-12 18:35:38 +0000617 instruction_hazard();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618}
619
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620#ifdef CONFIG_DMA_NONCOHERENT
621
622static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
623{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624 /* Catch bad driver code */
625 BUG_ON(size == 0);
626
Ralf Baechleff522052013-09-17 12:44:31 +0200627 preempt_disable();
Ralf Baechlefc5d2d22006-07-06 13:04:01 +0100628 if (cpu_has_inclusive_pcaches) {
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900629 if (size >= scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630 r4k_blast_scache();
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900631 else
632 blast_scache_range(addr, addr + size);
Yoichi Yuasa5596b0b2013-10-02 15:03:03 +0900633 preempt_enable();
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700634 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 return;
636 }
637
638 /*
639 * Either no secondary cache or the available caches don't have the
640 * subset property so we have to flush the primary caches
641 * explicitly
642 */
Ralf Baechle39b8d522008-04-28 17:14:26 +0100643 if (cpu_has_safe_index_cacheops && size >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 r4k_blast_dcache();
645 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646 R4600_HIT_CACHEOP_WAR_IMPL;
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900647 blast_dcache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648 }
Ralf Baechleff522052013-09-17 12:44:31 +0200649 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650
651 bc_wback_inv(addr, size);
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700652 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653}
654
655static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
656{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 /* Catch bad driver code */
658 BUG_ON(size == 0);
659
Ralf Baechleff522052013-09-17 12:44:31 +0200660 preempt_disable();
Ralf Baechlefc5d2d22006-07-06 13:04:01 +0100661 if (cpu_has_inclusive_pcaches) {
Atsushi Nemoto41700e72006-02-10 00:39:06 +0900662 if (size >= scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 r4k_blast_scache();
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000664 else {
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000665 /*
666 * There is no clearly documented alignment requirement
667 * for the cache instruction on MIPS processors and
668 * some processors, among them the RM5200 and RM7000
669 * QED processors will throw an address error for cache
Ralf Baechle70342282013-01-22 12:59:30 +0100670 * hit ops with insufficient alignment. Solved by
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000671 * aligning the address to cache line size.
672 */
Thomas Bogendoerfere9c33572007-11-26 23:40:01 +0100673 blast_inv_scache_range(addr, addr + size);
Ralf Baechlea8ca8b62009-01-11 18:44:49 +0000674 }
Yoichi Yuasa5596b0b2013-10-02 15:03:03 +0900675 preempt_enable();
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700676 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 return;
678 }
679
Ralf Baechle39b8d522008-04-28 17:14:26 +0100680 if (cpu_has_safe_index_cacheops && size >= dcache_size) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681 r4k_blast_dcache();
682 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 R4600_HIT_CACHEOP_WAR_IMPL;
Thomas Bogendoerfere9c33572007-11-26 23:40:01 +0100684 blast_inv_dcache_range(addr, addr + size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 }
Ralf Baechleff522052013-09-17 12:44:31 +0200686 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687
688 bc_inv(addr, size);
Kevin Cernekeed0023c42010-09-06 21:03:46 -0700689 __sync();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690}
691#endif /* CONFIG_DMA_NONCOHERENT */
692
693/*
694 * While we're protected against bad userland addresses we don't care
695 * very much about what happens in that case. Usually a segmentation
696 * fault will dump the process later on anyway ...
697 */
698static void local_r4k_flush_cache_sigtramp(void * arg)
699{
Thiemo Seufer02fe2c92005-09-09 19:45:41 +0000700 unsigned long ic_lsize = cpu_icache_line_size();
701 unsigned long dc_lsize = cpu_dcache_line_size();
702 unsigned long sc_lsize = cpu_scache_line_size();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 unsigned long addr = (unsigned long) arg;
704
705 R4600_HIT_CACHEOP_WAR_IMPL;
Chris Dearman73f40352006-06-20 18:06:52 +0100706 if (dc_lsize)
707 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
Ralf Baechle4debe4f2006-02-27 19:05:55 +0000708 if (!cpu_icache_snoops_remote_store && scache_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 protected_writeback_scache_line(addr & ~(sc_lsize - 1));
Chris Dearman73f40352006-06-20 18:06:52 +0100710 if (ic_lsize)
711 protected_flush_icache_line(addr & ~(ic_lsize - 1));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712 if (MIPS4K_ICACHE_REFILL_WAR) {
713 __asm__ __volatile__ (
714 ".set push\n\t"
715 ".set noat\n\t"
716 ".set mips3\n\t"
Ralf Baechle875d43e2005-09-03 15:56:16 -0700717#ifdef CONFIG_32BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718 "la $at,1f\n\t"
719#endif
Ralf Baechle875d43e2005-09-03 15:56:16 -0700720#ifdef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 "dla $at,1f\n\t"
722#endif
723 "cache %0,($at)\n\t"
724 "nop; nop; nop\n"
725 "1:\n\t"
726 ".set pop"
727 :
728 : "i" (Hit_Invalidate_I));
729 }
730 if (MIPS_CACHE_SYNC_WAR)
731 __asm__ __volatile__ ("sync");
732}
733
734static void r4k_flush_cache_sigtramp(unsigned long addr)
735{
Ralf Baechle48a26e62010-10-29 19:08:25 +0100736 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737}
738
739static void r4k_flush_icache_all(void)
740{
741 if (cpu_has_vtag_icache)
742 r4k_blast_icache();
743}
744
Ralf Baechled9cdc9012011-06-17 16:20:28 +0100745struct flush_kernel_vmap_range_args {
746 unsigned long vaddr;
747 int size;
748};
749
750static inline void local_r4k_flush_kernel_vmap_range(void *args)
751{
752 struct flush_kernel_vmap_range_args *vmra = args;
753 unsigned long vaddr = vmra->vaddr;
754 int size = vmra->size;
755
756 /*
757 * Aliases only affect the primary caches so don't bother with
758 * S-caches or T-caches.
759 */
760 if (cpu_has_safe_index_cacheops && size >= dcache_size)
761 r4k_blast_dcache();
762 else {
763 R4600_HIT_CACHEOP_WAR_IMPL;
764 blast_dcache_range(vaddr, vaddr + size);
765 }
766}
767
768static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size)
769{
770 struct flush_kernel_vmap_range_args args;
771
772 args.vaddr = (unsigned long) vaddr;
773 args.size = size;
774
775 r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args);
776}
777
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778static inline void rm7k_erratum31(void)
779{
780 const unsigned long ic_lsize = 32;
781 unsigned long addr;
782
783 /* RM7000 erratum #31. The icache is screwed at startup. */
784 write_c0_taglo(0);
785 write_c0_taghi(0);
786
787 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
788 __asm__ __volatile__ (
Thiemo Seuferd8748a32005-09-02 09:56:12 +0000789 ".set push\n\t"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700790 ".set noreorder\n\t"
791 ".set mips3\n\t"
792 "cache\t%1, 0(%0)\n\t"
793 "cache\t%1, 0x1000(%0)\n\t"
794 "cache\t%1, 0x2000(%0)\n\t"
795 "cache\t%1, 0x3000(%0)\n\t"
796 "cache\t%2, 0(%0)\n\t"
797 "cache\t%2, 0x1000(%0)\n\t"
798 "cache\t%2, 0x2000(%0)\n\t"
799 "cache\t%2, 0x3000(%0)\n\t"
800 "cache\t%1, 0(%0)\n\t"
801 "cache\t%1, 0x1000(%0)\n\t"
802 "cache\t%1, 0x2000(%0)\n\t"
803 "cache\t%1, 0x3000(%0)\n\t"
Thiemo Seuferd8748a32005-09-02 09:56:12 +0000804 ".set pop\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805 :
806 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
807 }
808}
809
Steven J. Hill006a8512012-06-26 04:11:03 +0000810static inline void alias_74k_erratum(struct cpuinfo_mips *c)
811{
Maciej W. Rozycki9213ad72013-09-18 19:08:15 +0100812 unsigned int imp = c->processor_id & PRID_IMP_MASK;
813 unsigned int rev = c->processor_id & PRID_REV_MASK;
814
Steven J. Hill006a8512012-06-26 04:11:03 +0000815 /*
816 * Early versions of the 74K do not update the cache tags on a
817 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG
818 * aliases. In this case it is better to treat the cache as always
819 * having aliases.
820 */
Maciej W. Rozycki9213ad72013-09-18 19:08:15 +0100821 switch (imp) {
822 case PRID_IMP_74K:
823 if (rev <= PRID_REV_ENCODE_332(2, 4, 0))
824 c->dcache.flags |= MIPS_CACHE_VTAG;
825 if (rev == PRID_REV_ENCODE_332(2, 4, 0))
826 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
827 break;
828 case PRID_IMP_1074K:
829 if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) {
830 c->dcache.flags |= MIPS_CACHE_VTAG;
831 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
832 }
833 break;
834 default:
835 BUG();
Steven J. Hill006a8512012-06-26 04:11:03 +0000836 }
837}
838
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000839static char *way_string[] = { NULL, "direct mapped", "2-way",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
841};
842
Paul Gortmaker078a55f2013-06-18 13:38:59 +0000843static void probe_pcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844{
845 struct cpuinfo_mips *c = &current_cpu_data;
846 unsigned int config = read_c0_config();
847 unsigned int prid = read_c0_prid();
848 unsigned long config1;
849 unsigned int lsize;
850
Ralf Baechle69f24d12013-09-17 10:25:47 +0200851 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852 case CPU_R4600: /* QED style two way caches? */
853 case CPU_R4700:
854 case CPU_R5000:
855 case CPU_NEVADA:
856 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
857 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
858 c->icache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900859 c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860
861 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
862 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
863 c->dcache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900864 c->dcache.waybit= __ffs(dcache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865
866 c->options |= MIPS_CPU_CACHE_CDEX_P;
867 break;
868
869 case CPU_R5432:
870 case CPU_R5500:
871 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
872 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
873 c->icache.ways = 2;
874 c->icache.waybit= 0;
875
876 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
877 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
878 c->dcache.ways = 2;
879 c->dcache.waybit = 0;
880
Shinya Kuribayashi58648102009-03-18 09:04:01 +0900881 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700882 break;
883
884 case CPU_TX49XX:
885 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
886 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
887 c->icache.ways = 4;
888 c->icache.waybit= 0;
889
890 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
891 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
892 c->dcache.ways = 4;
893 c->dcache.waybit = 0;
894
895 c->options |= MIPS_CPU_CACHE_CDEX_P;
Atsushi Nemotode862b42006-03-17 12:59:22 +0900896 c->options |= MIPS_CPU_PREFETCH;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897 break;
898
899 case CPU_R4000PC:
900 case CPU_R4000SC:
901 case CPU_R4000MC:
902 case CPU_R4400PC:
903 case CPU_R4400SC:
904 case CPU_R4400MC:
905 case CPU_R4300:
906 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
907 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
908 c->icache.ways = 1;
Ralf Baechle70342282013-01-22 12:59:30 +0100909 c->icache.waybit = 0; /* doesn't matter */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910
911 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
912 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
913 c->dcache.ways = 1;
914 c->dcache.waybit = 0; /* does not matter */
915
916 c->options |= MIPS_CPU_CACHE_CDEX_P;
917 break;
918
919 case CPU_R10000:
920 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -0400921 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700922 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
923 c->icache.linesz = 64;
924 c->icache.ways = 2;
925 c->icache.waybit = 0;
926
927 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
928 c->dcache.linesz = 32;
929 c->dcache.ways = 2;
930 c->dcache.waybit = 0;
931
932 c->options |= MIPS_CPU_PREFETCH;
933 break;
934
935 case CPU_VR4133:
Yoichi Yuasa2874fe52006-07-08 00:42:12 +0900936 write_c0_config(config & ~VR41_CONF_P4K);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937 case CPU_VR4131:
938 /* Workaround for cache instruction bug of VR4131 */
939 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
940 c->processor_id == 0x0c82U) {
Yoichi Yuasa4e8ab362006-07-04 22:59:41 +0900941 config |= 0x00400000U;
942 if (c->processor_id == 0x0c80U)
943 config |= VR41_CONF_BP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 write_c0_config(config);
Yoichi Yuasa1058ecd2006-07-08 00:42:01 +0900945 } else
946 c->options |= MIPS_CPU_CACHE_CDEX_P;
947
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
949 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
950 c->icache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900951 c->icache.waybit = __ffs(icache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952
953 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
954 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
955 c->dcache.ways = 2;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900956 c->dcache.waybit = __ffs(dcache_size/2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957 break;
958
959 case CPU_VR41XX:
960 case CPU_VR4111:
961 case CPU_VR4121:
962 case CPU_VR4122:
963 case CPU_VR4181:
964 case CPU_VR4181A:
965 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
966 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
967 c->icache.ways = 1;
Ralf Baechle70342282013-01-22 12:59:30 +0100968 c->icache.waybit = 0; /* doesn't matter */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969
970 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
971 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
972 c->dcache.ways = 1;
973 c->dcache.waybit = 0; /* does not matter */
974
975 c->options |= MIPS_CPU_CACHE_CDEX_P;
976 break;
977
978 case CPU_RM7000:
979 rm7k_erratum31();
980
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
982 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
983 c->icache.ways = 4;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900984 c->icache.waybit = __ffs(icache_size / c->icache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985
986 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
987 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
988 c->dcache.ways = 4;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +0900989 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991 c->options |= MIPS_CPU_CACHE_CDEX_P;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992 c->options |= MIPS_CPU_PREFETCH;
993 break;
994
Fuxin Zhang2a21c732007-06-06 14:52:43 +0800995 case CPU_LOONGSON2:
996 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
997 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
998 if (prid & 0x3)
999 c->icache.ways = 4;
1000 else
1001 c->icache.ways = 2;
1002 c->icache.waybit = 0;
1003
1004 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1005 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1006 if (prid & 0x3)
1007 c->dcache.ways = 4;
1008 else
1009 c->dcache.ways = 2;
1010 c->dcache.waybit = 0;
1011 break;
1012
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 default:
1014 if (!(config & MIPS_CONF_M))
1015 panic("Don't know how to probe P-caches on this cpu.");
1016
1017 /*
1018 * So we seem to be a MIPS32 or MIPS64 CPU
1019 * So let's probe the I-cache ...
1020 */
1021 config1 = read_c0_config1();
1022
Markos Chandras175cba82013-09-19 18:18:41 +01001023 lsize = (config1 >> 19) & 7;
1024
1025 /* IL == 7 is reserved */
1026 if (lsize == 7)
1027 panic("Invalid icache line size");
1028
1029 c->icache.linesz = lsize ? 2 << lsize : 0;
1030
Douglas Leungdc34b052012-07-19 09:11:13 +02001031 c->icache.sets = 32 << (((config1 >> 22) + 1) & 7);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032 c->icache.ways = 1 + ((config1 >> 16) & 7);
1033
1034 icache_size = c->icache.sets *
Ralf Baechle70342282013-01-22 12:59:30 +01001035 c->icache.ways *
1036 c->icache.linesz;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +09001037 c->icache.waybit = __ffs(icache_size/c->icache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038
1039 if (config & 0x8) /* VI bit */
1040 c->icache.flags |= MIPS_CACHE_VTAG;
1041
1042 /*
1043 * Now probe the MIPS32 / MIPS64 data cache.
1044 */
1045 c->dcache.flags = 0;
1046
Markos Chandras175cba82013-09-19 18:18:41 +01001047 lsize = (config1 >> 10) & 7;
1048
1049 /* DL == 7 is reserved */
1050 if (lsize == 7)
1051 panic("Invalid dcache line size");
1052
1053 c->dcache.linesz = lsize ? 2 << lsize : 0;
1054
Douglas Leungdc34b052012-07-19 09:11:13 +02001055 c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056 c->dcache.ways = 1 + ((config1 >> 7) & 7);
1057
1058 dcache_size = c->dcache.sets *
Ralf Baechle70342282013-01-22 12:59:30 +01001059 c->dcache.ways *
1060 c->dcache.linesz;
Atsushi Nemoto3c68da72006-04-08 01:33:31 +09001061 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062
1063 c->options |= MIPS_CPU_PREFETCH;
1064 break;
1065 }
1066
1067 /*
1068 * Processor configuration sanity check for the R4000SC erratum
Ralf Baechle70342282013-01-22 12:59:30 +01001069 * #5. With page sizes larger than 32kB there is no possibility
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070 * to get a VCE exception anymore so we don't care about this
1071 * misconfiguration. The case is rather theoretical anyway;
1072 * presumably no vendor is shipping his hardware in the "bad"
1073 * configuration.
1074 */
Maciej W. Rozycki8ff374b2013-09-17 16:58:10 +01001075 if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 &&
1076 (prid & PRID_REV_MASK) < PRID_REV_R4400 &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077 !(config & CONF_SC) && c->icache.linesz != 16 &&
1078 PAGE_SIZE <= 0x8000)
1079 panic("Improper R4000SC processor configuration detected");
1080
1081 /* compute a couple of other cache variables */
1082 c->icache.waysize = icache_size / c->icache.ways;
1083 c->dcache.waysize = dcache_size / c->dcache.ways;
1084
Chris Dearman73f40352006-06-20 18:06:52 +01001085 c->icache.sets = c->icache.linesz ?
1086 icache_size / (c->icache.linesz * c->icache.ways) : 0;
1087 c->dcache.sets = c->dcache.linesz ?
1088 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089
1090 /*
1091 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB
1092 * 2-way virtually indexed so normally would suffer from aliases. So
1093 * normally they'd suffer from aliases but magic in the hardware deals
1094 * with that for us so we don't need to take care ourselves.
1095 */
Ralf Baechle69f24d12013-09-17 10:25:47 +02001096 switch (current_cpu_type()) {
Ralf Baechlea95970f2005-02-07 21:41:32 +00001097 case CPU_20KC:
Ralf Baechle505403b2005-02-07 21:53:39 +00001098 case CPU_25KF:
Ralf Baechle641e97f2007-10-11 23:46:05 +01001099 case CPU_SB1:
1100 case CPU_SB1A:
Jayachandran Cefa0f812011-05-07 01:36:21 +05301101 case CPU_XLR:
Atsushi Nemotode628932006-03-13 18:23:03 +09001102 c->dcache.flags |= MIPS_CACHE_PINDEX;
Ralf Baechle641e97f2007-10-11 23:46:05 +01001103 break;
1104
Ralf Baechled1e344e2005-02-04 15:51:26 +00001105 case CPU_R10000:
1106 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -04001107 case CPU_R14000:
Ralf Baechled1e344e2005-02-04 15:51:26 +00001108 break;
Ralf Baechle641e97f2007-10-11 23:46:05 +01001109
Steven J. Hill113c62d2012-07-06 23:56:00 +02001110 case CPU_M14KC:
Steven J. Hillf8fa4812012-12-07 03:51:35 +00001111 case CPU_M14KEC:
Ralf Baechled1e344e2005-02-04 15:51:26 +00001112 case CPU_24K:
Nigel Stephens98a41de2006-04-27 15:50:32 +01001113 case CPU_34K:
Ralf Baechle2e78ae32006-06-23 18:48:21 +01001114 case CPU_74K:
Ralf Baechle39b8d522008-04-28 17:14:26 +01001115 case CPU_1004K:
Leonid Yegoshin26ab96d2013-11-27 10:07:53 +00001116 case CPU_INTERAPTIV:
Leonid Yegoshin708ac4b2013-11-14 16:12:27 +00001117 case CPU_PROAPTIV:
Ralf Baechle69f24d12013-09-17 10:25:47 +02001118 if (current_cpu_type() == CPU_74K)
Steven J. Hill006a8512012-06-26 04:11:03 +00001119 alias_74k_erratum(c);
Ralf Baechlebeab3752006-06-19 21:56:25 +01001120 if ((read_c0_config7() & (1 << 16))) {
1121 /* effectively physically indexed dcache,
1122 thus no virtual aliases. */
1123 c->dcache.flags |= MIPS_CACHE_PINDEX;
1124 break;
1125 }
Ralf Baechled1e344e2005-02-04 15:51:26 +00001126 default:
Ralf Baechlebeab3752006-06-19 21:56:25 +01001127 if (c->dcache.waysize > PAGE_SIZE)
1128 c->dcache.flags |= MIPS_CACHE_ALIASES;
Ralf Baechled1e344e2005-02-04 15:51:26 +00001129 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001130
Ralf Baechle69f24d12013-09-17 10:25:47 +02001131 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132 case CPU_20KC:
1133 /*
1134 * Some older 20Kc chips doesn't have the 'VI' bit in
1135 * the config register.
1136 */
1137 c->icache.flags |= MIPS_CACHE_VTAG;
1138 break;
1139
Manuel Lauss270717a2009-03-25 17:49:28 +01001140 case CPU_ALCHEMY:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001141 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1142 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143
Ralf Baechle14bd8c02013-09-25 18:21:26 +02001144 case CPU_LOONGSON2:
1145 /*
1146 * LOONGSON2 has 4 way icache, but when using indexed cache op,
1147 * one op will act on all 4 ways
1148 */
1149 c->icache.ways = 1;
1150 }
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001151
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1153 icache_size >> 10,
Ralf Baechle7fc73162009-04-01 16:11:53 +02001154 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001155 way_string[c->icache.ways], c->icache.linesz);
1156
Ralf Baechle64bfca52007-10-15 16:35:45 +01001157 printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
1158 dcache_size >> 10, way_string[c->dcache.ways],
1159 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT",
1160 (c->dcache.flags & MIPS_CACHE_ALIASES) ?
1161 "cache aliases" : "no aliases",
1162 c->dcache.linesz);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163}
1164
1165/*
1166 * If you even _breathe_ on this function, look at the gcc output and make sure
1167 * it does not pop things on and off the stack for the cache sizing loop that
1168 * executes in KSEG1 space or else you will crash and burn badly. You have
1169 * been warned.
1170 */
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001171static int probe_scache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173 unsigned long flags, addr, begin, end, pow2;
1174 unsigned int config = read_c0_config();
1175 struct cpuinfo_mips *c = &current_cpu_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176
1177 if (config & CONF_SC)
1178 return 0;
1179
Ralf Baechlee001e522007-07-28 12:45:47 +01001180 begin = (unsigned long) &_stext;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001181 begin &= ~((4 * 1024 * 1024) - 1);
1182 end = begin + (4 * 1024 * 1024);
1183
1184 /*
1185 * This is such a bitch, you'd think they would make it easy to do
1186 * this. Away you daemons of stupidity!
1187 */
1188 local_irq_save(flags);
1189
1190 /* Fill each size-multiple cache line with a valid tag. */
1191 pow2 = (64 * 1024);
1192 for (addr = begin; addr < end; addr = (begin + pow2)) {
1193 unsigned long *p = (unsigned long *) addr;
1194 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1195 pow2 <<= 1;
1196 }
1197
1198 /* Load first line with zero (therefore invalid) tag. */
1199 write_c0_taglo(0);
1200 write_c0_taghi(0);
1201 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1202 cache_op(Index_Store_Tag_I, begin);
1203 cache_op(Index_Store_Tag_D, begin);
1204 cache_op(Index_Store_Tag_SD, begin);
1205
1206 /* Now search for the wrap around point. */
1207 pow2 = (128 * 1024);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1209 cache_op(Index_Load_Tag_SD, addr);
1210 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1211 if (!read_c0_taglo())
1212 break;
1213 pow2 <<= 1;
1214 }
1215 local_irq_restore(flags);
1216 addr -= begin;
1217
1218 scache_size = addr;
1219 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1220 c->scache.ways = 1;
1221 c->dcache.waybit = 0; /* does not matter */
1222
1223 return 1;
1224}
1225
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001226static void __init loongson2_sc_init(void)
1227{
1228 struct cpuinfo_mips *c = &current_cpu_data;
1229
1230 scache_size = 512*1024;
1231 c->scache.linesz = 32;
1232 c->scache.ways = 4;
1233 c->scache.waybit = 0;
1234 c->scache.waysize = scache_size / (c->scache.ways);
1235 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1236 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1237 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1238
1239 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1240}
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001241
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242extern int r5k_sc_init(void);
1243extern int rm7k_sc_init(void);
Chris Dearman9318c512006-06-20 17:15:20 +01001244extern int mips_sc_init(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001245
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001246static void setup_scache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247{
1248 struct cpuinfo_mips *c = &current_cpu_data;
1249 unsigned int config = read_c0_config();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250 int sc_present = 0;
1251
1252 /*
1253 * Do the probing thing on R4000SC and R4400SC processors. Other
1254 * processors don't have a S-cache that would be relevant to the
Joe Perches603e82e2008-02-03 16:54:53 +02001255 * Linux memory management.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256 */
Ralf Baechle69f24d12013-09-17 10:25:47 +02001257 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001258 case CPU_R4000SC:
1259 case CPU_R4000MC:
1260 case CPU_R4400SC:
1261 case CPU_R4400MC:
Thiemo Seuferba5187d2005-04-25 16:36:23 +00001262 sc_present = run_uncached(probe_scache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263 if (sc_present)
1264 c->options |= MIPS_CPU_CACHE_CDEX_S;
1265 break;
1266
1267 case CPU_R10000:
1268 case CPU_R12000:
Kumba44d921b2006-05-16 22:23:59 -04001269 case CPU_R14000:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1271 c->scache.linesz = 64 << ((config >> 13) & 1);
1272 c->scache.ways = 2;
1273 c->scache.waybit= 0;
1274 sc_present = 1;
1275 break;
1276
1277 case CPU_R5000:
1278 case CPU_NEVADA:
1279#ifdef CONFIG_R5000_CPU_SCACHE
1280 r5k_sc_init();
1281#endif
Ralf Baechle70342282013-01-22 12:59:30 +01001282 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001283
1284 case CPU_RM7000:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285#ifdef CONFIG_RM7000_CPU_SCACHE
1286 rm7k_sc_init();
1287#endif
1288 return;
1289
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001290 case CPU_LOONGSON2:
1291 loongson2_sc_init();
1292 return;
Ralf Baechle14bd8c02013-09-25 18:21:26 +02001293
Jayachandran Ca3d4fb22011-11-16 00:21:20 +00001294 case CPU_XLP:
1295 /* don't need to worry about L2, fully coherent */
1296 return;
Fuxin Zhang2a21c732007-06-06 14:52:43 +08001297
Linus Torvalds1da177e2005-04-16 15:20:36 -07001298 default:
Deng-Cheng Zhuadb37892013-04-01 18:14:28 +00001299 if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 |
1300 MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M64R2)) {
Chris Dearman9318c512006-06-20 17:15:20 +01001301#ifdef CONFIG_MIPS_CPU_SCACHE
1302 if (mips_sc_init ()) {
1303 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
1304 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
1305 scache_size >> 10,
1306 way_string[c->scache.ways], c->scache.linesz);
1307 }
1308#else
1309 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1310 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1311#endif
1312 return;
1313 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001314 sc_present = 0;
1315 }
1316
1317 if (!sc_present)
1318 return;
1319
Linus Torvalds1da177e2005-04-16 15:20:36 -07001320 /* compute a couple of other cache variables */
1321 c->scache.waysize = scache_size / c->scache.ways;
1322
1323 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1324
1325 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1326 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1327
Ralf Baechlefc5d2d22006-07-06 13:04:01 +01001328 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001329}
1330
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001331void au1x00_fixup_config_od(void)
1332{
1333 /*
1334 * c0_config.od (bit 19) was write only (and read as 0)
1335 * on the early revisions of Alchemy SOCs. It disables the bus
1336 * transaction overlapping and needs to be set to fix various errata.
1337 */
1338 switch (read_c0_prid()) {
1339 case 0x00030100: /* Au1000 DA */
1340 case 0x00030201: /* Au1000 HA */
1341 case 0x00030202: /* Au1000 HB */
1342 case 0x01030200: /* Au1500 AB */
1343 /*
1344 * Au1100 errata actually keeps silence about this bit, so we set it
1345 * just in case for those revisions that require it to be set according
Manuel Lauss270717a2009-03-25 17:49:28 +01001346 * to the (now gone) cpu table.
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001347 */
1348 case 0x02030200: /* Au1100 AB */
1349 case 0x02030201: /* Au1100 BA */
1350 case 0x02030202: /* Au1100 BC */
1351 set_c0_config(1 << 19);
1352 break;
1353 }
1354}
1355
Ralf Baechle89052bd2008-06-12 17:26:02 +01001356/* CP0 hazard avoidance. */
1357#define NXP_BARRIER() \
1358 __asm__ __volatile__( \
1359 ".set noreorder\n\t" \
1360 "nop; nop; nop; nop; nop; nop;\n\t" \
1361 ".set reorder\n\t")
1362
1363static void nxp_pr4450_fixup_config(void)
1364{
1365 unsigned long config0;
1366
1367 config0 = read_c0_config();
1368
1369 /* clear all three cache coherency fields */
1370 config0 &= ~(0x7 | (7 << 25) | (7 << 28));
1371 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) |
1372 ((_page_cachable_default >> _CACHE_SHIFT) << 25) |
1373 ((_page_cachable_default >> _CACHE_SHIFT) << 28));
1374 write_c0_config(config0);
1375 NXP_BARRIER();
1376}
1377
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001378static int cca = -1;
Chris Dearman35133692007-09-19 00:58:24 +01001379
1380static int __init cca_setup(char *str)
1381{
1382 get_option(&str, &cca);
1383
Shane McDonaldb5b64f22012-06-14 02:26:40 +00001384 return 0;
Chris Dearman35133692007-09-19 00:58:24 +01001385}
1386
Shane McDonaldb5b64f22012-06-14 02:26:40 +00001387early_param("cca", cca_setup);
Chris Dearman35133692007-09-19 00:58:24 +01001388
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001389static void coherency_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001390{
Chris Dearman35133692007-09-19 00:58:24 +01001391 if (cca < 0 || cca > 7)
1392 cca = read_c0_config() & CONF_CM_CMASK;
1393 _page_cachable_default = cca << _CACHE_SHIFT;
1394
1395 pr_debug("Using cache attribute %d\n", cca);
1396 change_c0_config(CONF_CM_CMASK, cca);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397
1398 /*
1399 * c0_status.cu=0 specifies that updates by the sc instruction use
1400 * the coherency mode specified by the TLB; 1 means cachable
1401 * coherent update on write will be used. Not all processors have
1402 * this bit and; some wire it to zero, others like Toshiba had the
1403 * silly idea of putting something else there ...
1404 */
Ralf Baechle10cc3522007-10-11 23:46:15 +01001405 switch (current_cpu_type()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406 case CPU_R4000PC:
1407 case CPU_R4000SC:
1408 case CPU_R4000MC:
1409 case CPU_R4400PC:
1410 case CPU_R4400SC:
1411 case CPU_R4400MC:
1412 clear_c0_config(CONF_CU);
1413 break;
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001414 /*
Ralf Baechledf586d52006-08-01 23:42:30 +01001415 * We need to catch the early Alchemy SOCs with
Manuel Lauss270717a2009-03-25 17:49:28 +01001416 * the write-only co_config.od bit and set it back to one on:
1417 * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001418 */
Manuel Lauss270717a2009-03-25 17:49:28 +01001419 case CPU_ALCHEMY:
Sergei Shtylyov9370b352006-05-26 19:44:54 +04001420 au1x00_fixup_config_od();
1421 break;
Ralf Baechle89052bd2008-06-12 17:26:02 +01001422
1423 case PRID_IMP_PR4450:
1424 nxp_pr4450_fixup_config();
1425 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426 }
1427}
1428
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001429static void r4k_cache_error_setup(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430{
Ralf Baechle641e97f2007-10-11 23:46:05 +01001431 extern char __weak except_vec2_generic;
1432 extern char __weak except_vec2_sb1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001433
Ralf Baechle69f24d12013-09-17 10:25:47 +02001434 switch (current_cpu_type()) {
Ralf Baechle641e97f2007-10-11 23:46:05 +01001435 case CPU_SB1:
1436 case CPU_SB1A:
1437 set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
1438 break;
1439
1440 default:
1441 set_uncached_handler(0x100, &except_vec2_generic, 0x80);
1442 break;
1443 }
David Daney9cd9669b2012-05-15 00:04:49 -07001444}
1445
Paul Gortmaker078a55f2013-06-18 13:38:59 +00001446void r4k_cache_init(void)
David Daney9cd9669b2012-05-15 00:04:49 -07001447{
1448 extern void build_clear_page(void);
1449 extern void build_copy_page(void);
1450 struct cpuinfo_mips *c = &current_cpu_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001451
1452 probe_pcache();
1453 setup_scache();
1454
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455 r4k_blast_dcache_page_setup();
1456 r4k_blast_dcache_page_indexed_setup();
1457 r4k_blast_dcache_setup();
1458 r4k_blast_icache_page_setup();
1459 r4k_blast_icache_page_indexed_setup();
1460 r4k_blast_icache_setup();
1461 r4k_blast_scache_page_setup();
1462 r4k_blast_scache_page_indexed_setup();
1463 r4k_blast_scache_setup();
1464
1465 /*
1466 * Some MIPS32 and MIPS64 processors have physically indexed caches.
1467 * This code supports virtually indexed processors and will be
1468 * unnecessarily inefficient on physically indexed processors.
1469 */
Chris Dearman73f40352006-06-20 18:06:52 +01001470 if (c->dcache.linesz)
1471 shm_align_mask = max_t( unsigned long,
1472 c->dcache.sets * c->dcache.linesz - 1,
1473 PAGE_SIZE - 1);
1474 else
1475 shm_align_mask = PAGE_SIZE-1;
Ralf Baechle9c5a3d72008-04-05 15:13:23 +01001476
1477 __flush_cache_vmap = r4k__flush_cache_vmap;
1478 __flush_cache_vunmap = r4k__flush_cache_vunmap;
1479
Ralf Baechledb813fe2007-09-27 18:26:43 +01001480 flush_cache_all = cache_noop;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001481 __flush_cache_all = r4k___flush_cache_all;
1482 flush_cache_mm = r4k_flush_cache_mm;
1483 flush_cache_page = r4k_flush_cache_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001484 flush_cache_range = r4k_flush_cache_range;
1485
Ralf Baechled9cdc9012011-06-17 16:20:28 +01001486 __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range;
1487
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488 flush_cache_sigtramp = r4k_flush_cache_sigtramp;
1489 flush_icache_all = r4k_flush_icache_all;
Ralf Baechle7e3bfc72006-04-05 20:42:04 +01001490 local_flush_data_cache_page = local_r4k_flush_data_cache_page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001491 flush_data_cache_page = r4k_flush_data_cache_page;
1492 flush_icache_range = r4k_flush_icache_range;
Thomas Bogendoerfere0cee3e2008-08-04 20:53:57 +02001493 local_flush_icache_range = local_r4k_flush_icache_range;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001494
Ralf Baechle39b8d522008-04-28 17:14:26 +01001495#if defined(CONFIG_DMA_NONCOHERENT)
1496 if (coherentio) {
1497 _dma_cache_wback_inv = (void *)cache_noop;
1498 _dma_cache_wback = (void *)cache_noop;
1499 _dma_cache_inv = (void *)cache_noop;
1500 } else {
1501 _dma_cache_wback_inv = r4k_dma_cache_wback_inv;
1502 _dma_cache_wback = r4k_dma_cache_wback_inv;
1503 _dma_cache_inv = r4k_dma_cache_inv;
1504 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001505#endif
1506
Linus Torvalds1da177e2005-04-16 15:20:36 -07001507 build_clear_page();
1508 build_copy_page();
Steven J. Hillb6d92b42013-03-25 13:47:29 -05001509
1510 /*
1511 * We want to run CMP kernels on core with and without coherent
1512 * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether
1513 * or not to flush caches.
1514 */
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00001515 local_r4k___flush_cache_all(NULL);
Steven J. Hillb6d92b42013-03-25 13:47:29 -05001516
Ralf Baechle1d40cfc2005-07-15 15:23:23 +00001517 coherency_setup();
David Daney9cd9669b2012-05-15 00:04:49 -07001518 board_cache_error_setup = r4k_cache_error_setup;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519}