blob: 69bb818fdd798b8d50dff44a46a9941272972a92 [file] [log] [blame]
Adrian Bunkb00dc832008-05-19 16:52:27 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * arch/sparc64/mm/fault.c: Page fault handlers for the 64-bit Sparc.
3 *
David S. Miller4fe3ebe2008-07-17 22:11:32 -07004 * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005 * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
6 */
7
8#include <asm/head.h>
9
10#include <linux/string.h>
11#include <linux/types.h>
12#include <linux/sched.h>
13#include <linux/ptrace.h>
14#include <linux/mman.h>
15#include <linux/signal.h>
16#include <linux/mm.h>
17#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/init.h>
David S. Millera084b662010-01-20 03:04:14 -080019#include <linux/perf_event.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/interrupt.h>
Prasanna S Panchamukhi05e14cb2005-09-06 15:19:30 -070021#include <linux/kprobes.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070022#include <linux/kdebug.h>
David S. Millereeabac72009-02-02 22:08:15 -080023#include <linux/percpu.h>
Kirill Tkhai812cb832013-09-14 16:02:11 +040024#include <linux/context_tracking.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
26#include <asm/page.h>
27#include <asm/pgtable.h>
28#include <asm/openprom.h>
29#include <asm/oplib.h>
30#include <asm/uaccess.h>
31#include <asm/asi.h>
32#include <asm/lsu.h>
33#include <asm/sections.h>
David S. Miller7a1ac522006-03-16 02:02:32 -080034#include <asm/mmu_context.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035
David S. Miller4b177642010-03-01 00:02:23 -080036int show_unhandled_signals = 1;
37
David S. Miller4ed5d5e2009-12-10 18:08:29 -080038static inline __kprobes int notify_page_fault(struct pt_regs *regs)
Anil S Keshavamurthyd98f8f02006-06-26 00:25:27 -070039{
David S. Miller127cda12007-05-08 18:25:14 -070040 int ret = 0;
Anil S Keshavamurthyd98f8f02006-06-26 00:25:27 -070041
David S. Miller127cda12007-05-08 18:25:14 -070042 /* kprobe_running() needs smp_processor_id() */
David S. Miller135d0822009-12-10 18:02:19 -080043 if (kprobes_built_in() && !user_mode(regs)) {
David S. Miller127cda12007-05-08 18:25:14 -070044 preempt_disable();
45 if (kprobe_running() && kprobe_fault_handler(regs, 0))
46 ret = 1;
47 preempt_enable();
48 }
49 return ret;
Anil S Keshavamurthyd98f8f02006-06-26 00:25:27 -070050}
Anil S Keshavamurthyd98f8f02006-06-26 00:25:27 -070051
Prasanna S Panchamukhi05e14cb2005-09-06 15:19:30 -070052static void __kprobes unhandled_fault(unsigned long address,
53 struct task_struct *tsk,
54 struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -070055{
56 if ((unsigned long) address < PAGE_SIZE) {
57 printk(KERN_ALERT "Unable to handle kernel NULL "
58 "pointer dereference\n");
59 } else {
60 printk(KERN_ALERT "Unable to handle kernel paging request "
61 "at virtual address %016lx\n", (unsigned long)address);
62 }
63 printk(KERN_ALERT "tsk->{mm,active_mm}->context = %016lx\n",
64 (tsk->mm ?
65 CTX_HWBITS(tsk->mm->context) :
66 CTX_HWBITS(tsk->active_mm->context)));
67 printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %016lx\n",
68 (tsk->mm ? (unsigned long) tsk->mm->pgd :
69 (unsigned long) tsk->active_mm->pgd));
Linus Torvalds1da177e2005-04-16 15:20:36 -070070 die_if_kernel("Oops", regs);
71}
72
David S. Miller4ed5d5e2009-12-10 18:08:29 -080073static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr)
Linus Torvalds1da177e2005-04-16 15:20:36 -070074{
Linus Torvalds1da177e2005-04-16 15:20:36 -070075 printk(KERN_CRIT "OOPS: Bogus kernel PC [%016lx] in fault handler\n",
76 regs->tpc);
David S. Millereb398d12006-07-22 01:12:09 -070077 printk(KERN_CRIT "OOPS: RPC [%016lx]\n", regs->u_regs[15]);
David S. Miller4fe3ebe2008-07-17 22:11:32 -070078 printk("OOPS: RPC <%pS>\n", (void *) regs->u_regs[15]);
David S. Millerbf941d62006-02-13 18:07:45 -080079 printk(KERN_CRIT "OOPS: Fault was to vaddr[%lx]\n", vaddr);
David S. Millerc1f193a2007-07-30 00:17:12 -070080 dump_stack();
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 unhandled_fault(regs->tpc, current, regs);
82}
83
84/*
85 * We now make sure that mmap_sem is held in all paths that call
86 * this. Additionally, to prevent kswapd from ripping ptes from
87 * under us, raise interrupts around the time that we look at the
88 * pte, kswapd will have to wait to get his smp ipi response from
Hugh Dickinsda160542005-11-08 10:00:55 -080089 * us. vmtruncate likewise. This saves us having to get pte lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 */
91static unsigned int get_user_insn(unsigned long tpc)
92{
93 pgd_t *pgdp = pgd_offset(current->mm, tpc);
94 pud_t *pudp;
95 pmd_t *pmdp;
96 pte_t *ptep, pte;
97 unsigned long pa;
98 u32 insn = 0;
99 unsigned long pstate;
100
101 if (pgd_none(*pgdp))
102 goto outret;
103 pudp = pud_offset(pgdp, tpc);
104 if (pud_none(*pudp))
105 goto outret;
106 pmdp = pmd_offset(pudp, tpc);
107 if (pmd_none(*pmdp))
108 goto outret;
109
110 /* This disables preemption for us as well. */
111 __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
112 __asm__ __volatile__("wrpr %0, %1, %%pstate"
113 : : "r" (pstate), "i" (PSTATE_IE));
114 ptep = pte_offset_map(pmdp, tpc);
115 pte = *ptep;
116 if (!pte_present(pte))
117 goto out;
118
David S. Millerc4bce902006-02-11 21:57:54 -0800119 pa = (pte_pfn(pte) << PAGE_SHIFT);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120 pa += (tpc & ~PAGE_MASK);
121
122 /* Use phys bypass so we don't pollute dtlb/dcache. */
123 __asm__ __volatile__("lduwa [%1] %2, %0"
124 : "=r" (insn)
125 : "r" (pa), "i" (ASI_PHYS_USE_EC));
126
127out:
128 pte_unmap(ptep);
129 __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
130outret:
131 return insn;
132}
133
David S. Miller4b177642010-03-01 00:02:23 -0800134static inline void
135show_signal_msg(struct pt_regs *regs, int sig, int code,
136 unsigned long address, struct task_struct *tsk)
137{
138 if (!unhandled_signal(tsk, sig))
139 return;
140
141 if (!printk_ratelimit())
142 return;
143
144 printk("%s%s[%d]: segfault at %lx ip %p (rpc %p) sp %p error %x",
145 task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
146 tsk->comm, task_pid_nr(tsk), address,
147 (void *)regs->tpc, (void *)regs->u_regs[UREG_I7],
148 (void *)regs->u_regs[UREG_FP], code);
149
150 print_vma_addr(KERN_CONT " in ", regs->tpc);
151
152 printk(KERN_CONT "\n");
153}
154
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155static void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
156 unsigned int insn, int fault_code)
157{
David S. Miller4b177642010-03-01 00:02:23 -0800158 unsigned long addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 siginfo_t info;
160
161 info.si_code = code;
162 info.si_signo = sig;
163 info.si_errno = 0;
164 if (fault_code & FAULT_CODE_ITLB)
David S. Miller4b177642010-03-01 00:02:23 -0800165 addr = regs->tpc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166 else
David S. Miller4b177642010-03-01 00:02:23 -0800167 addr = compute_effective_address(regs, insn, 0);
168 info.si_addr = (void __user *) addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 info.si_trapno = 0;
David S. Miller4b177642010-03-01 00:02:23 -0800170
171 if (unlikely(show_unhandled_signals))
172 show_signal_msg(regs, sig, code, addr, current);
173
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174 force_sig_info(sig, &info, current);
175}
176
177extern int handle_ldf_stq(u32, struct pt_regs *);
178extern int handle_ld_nf(u32, struct pt_regs *);
179
180static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn)
181{
182 if (!insn) {
183 if (!regs->tpc || (regs->tpc & 0x3))
184 return 0;
185 if (regs->tstate & TSTATE_PRIV) {
186 insn = *(unsigned int *) regs->tpc;
187 } else {
188 insn = get_user_insn(regs->tpc);
189 }
190 }
191 return insn;
192}
193
David S. Miller4ed5d5e2009-12-10 18:08:29 -0800194static void __kprobes do_kernel_fault(struct pt_regs *regs, int si_code,
195 int fault_code, unsigned int insn,
196 unsigned long address)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 unsigned char asi = ASI_P;
199
200 if ((!insn) && (regs->tstate & TSTATE_PRIV))
201 goto cannot_handle;
202
203 /* If user insn could be read (thus insn is zero), that
204 * is fine. We will just gun down the process with a signal
205 * in that case.
206 */
207
208 if (!(fault_code & (FAULT_CODE_WRITE|FAULT_CODE_ITLB)) &&
209 (insn & 0xc0800000) == 0xc0800000) {
210 if (insn & 0x2000)
211 asi = (regs->tstate >> 24);
212 else
213 asi = (insn >> 5);
214 if ((asi & 0xf2) == 0x82) {
215 if (insn & 0x1000000) {
216 handle_ldf_stq(insn, regs);
217 } else {
218 /* This was a non-faulting load. Just clear the
219 * destination register(s) and continue with the next
220 * instruction. -jj
221 */
222 handle_ld_nf(insn, regs);
223 }
224 return;
225 }
226 }
227
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 /* Is this in ex_table? */
229 if (regs->tstate & TSTATE_PRIV) {
David S. Miller8cf14af2005-09-28 20:21:11 -0700230 const struct exception_table_entry *entry;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231
David S. Miller622eaec2008-02-26 17:30:02 -0800232 entry = search_exception_tables(regs->tpc);
233 if (entry) {
David S. Miller8cf14af2005-09-28 20:21:11 -0700234 regs->tpc = entry->fixup;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235 regs->tnpc = regs->tpc + 4;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236 return;
237 }
238 } else {
239 /* The si_code was set to make clear whether
240 * this was a SEGV_MAPERR or SEGV_ACCERR fault.
241 */
242 do_fault_siginfo(si_code, SIGSEGV, regs, insn, fault_code);
243 return;
244 }
245
246cannot_handle:
247 unhandled_fault (address, current, regs);
248}
249
David S. Miller4ed5d5e2009-12-10 18:08:29 -0800250static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs)
David S. Miller9b026052009-02-03 16:28:23 -0800251{
252 static int times;
253
254 if (times++ < 10)
255 printk(KERN_ERR "FAULT[%s:%d]: 32-bit process reports "
256 "64-bit TPC [%lx]\n",
257 current->comm, current->pid,
258 regs->tpc);
259 show_regs(regs);
260}
261
David S. Miller4ed5d5e2009-12-10 18:08:29 -0800262static void noinline __kprobes bogus_32bit_fault_address(struct pt_regs *regs,
263 unsigned long addr)
David S. Miller9b026052009-02-03 16:28:23 -0800264{
265 static int times;
266
267 if (times++ < 10)
268 printk(KERN_ERR "FAULT[%s:%d]: 32-bit process "
269 "reports 64-bit fault address [%lx]\n",
270 current->comm, current->pid, addr);
271 show_regs(regs);
272}
273
Prasanna S Panchamukhi05e14cb2005-09-06 15:19:30 -0700274asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275{
Kirill Tkhai812cb832013-09-14 16:02:11 +0400276 enum ctx_state prev_state = exception_enter();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 struct mm_struct *mm = current->mm;
278 struct vm_area_struct *vma;
279 unsigned int insn = 0;
Nick Piggin83c54072007-07-19 01:47:05 -0700280 int si_code, fault_code, fault;
David S. Miller7a1ac522006-03-16 02:02:32 -0800281 unsigned long address, mm_rss;
Kautuk Consul7358e512012-03-26 06:40:49 +0000282 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283
284 fault_code = get_thread_fault_code();
285
David S. Miller127cda12007-05-08 18:25:14 -0700286 if (notify_page_fault(regs))
Kirill Tkhai812cb832013-09-14 16:02:11 +0400287 goto exit_exception;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288
289 si_code = SEGV_MAPERR;
290 address = current_thread_info()->fault_address;
291
292 if ((fault_code & FAULT_CODE_ITLB) &&
293 (fault_code & FAULT_CODE_DTLB))
294 BUG();
295
David S. Millereeabac72009-02-02 22:08:15 -0800296 if (test_thread_flag(TIF_32BIT)) {
David S. Miller9b026052009-02-03 16:28:23 -0800297 if (!(regs->tstate & TSTATE_PRIV)) {
298 if (unlikely((regs->tpc >> 32) != 0)) {
299 bogus_32bit_fault_tpc(regs);
300 goto intr_or_no_mm;
301 }
302 }
303 if (unlikely((address >> 32) != 0)) {
304 bogus_32bit_fault_address(regs, address);
305 goto intr_or_no_mm;
306 }
David S. Millereeabac72009-02-02 22:08:15 -0800307 }
308
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 if (regs->tstate & TSTATE_PRIV) {
David S. Miller9b026052009-02-03 16:28:23 -0800310 unsigned long tpc = regs->tpc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311
312 /* Sanity check the PC. */
David S. Millerbe717162008-02-28 20:38:15 -0800313 if ((tpc >= KERNBASE && tpc < (unsigned long) __init_end) ||
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 (tpc >= MODULES_VADDR && tpc < MODULES_END)) {
315 /* Valid, no problems... */
316 } else {
David S. Millerbf941d62006-02-13 18:07:45 -0800317 bad_kernel_pc(regs, address);
Kirill Tkhai812cb832013-09-14 16:02:11 +0400318 goto exit_exception;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319 }
Johannes Weiner759496b2013-09-12 15:13:39 -0700320 } else
321 flags |= FAULT_FLAG_USER;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322
323 /*
324 * If we're in an interrupt or have no user
325 * context, we must not take the fault..
326 */
327 if (in_atomic() || !mm)
328 goto intr_or_no_mm;
329
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200330 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
David S. Millera084b662010-01-20 03:04:14 -0800331
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 if (!down_read_trylock(&mm->mmap_sem)) {
333 if ((regs->tstate & TSTATE_PRIV) &&
334 !search_exception_tables(regs->tpc)) {
335 insn = get_fault_insn(regs, insn);
336 goto handle_kernel_fault;
337 }
Kautuk Consul7358e512012-03-26 06:40:49 +0000338
339retry:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 down_read(&mm->mmap_sem);
341 }
342
343 vma = find_vma(mm, address);
344 if (!vma)
345 goto bad_area;
346
347 /* Pure DTLB misses do not tell us whether the fault causing
348 * load/store/atomic was a write or not, it only says that there
349 * was no match. So in such a case we (carefully) read the
350 * instruction to try and figure this out. It's an optimization
351 * so it's ok if we can't do this.
352 *
353 * Special hack, window spill/fill knows the exact fault type.
354 */
355 if (((fault_code &
356 (FAULT_CODE_DTLB | FAULT_CODE_WRITE | FAULT_CODE_WINFIXUP)) == FAULT_CODE_DTLB) &&
357 (vma->vm_flags & VM_WRITE) != 0) {
358 insn = get_fault_insn(regs, 0);
359 if (!insn)
360 goto continue_fault;
David S. Miller73c50a22006-03-28 13:32:24 -0800361 /* All loads, stores and atomics have bits 30 and 31 both set
362 * in the instruction. Bit 21 is set in all stores, but we
363 * have to avoid prefetches which also have bit 21 set.
364 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 if ((insn & 0xc0200000) == 0xc0200000 &&
David S. Miller73c50a22006-03-28 13:32:24 -0800366 (insn & 0x01780000) != 0x01680000) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 /* Don't bother updating thread struct value,
368 * because update_mmu_cache only cares which tlb
369 * the access came from.
370 */
371 fault_code |= FAULT_CODE_WRITE;
372 }
373 }
374continue_fault:
375
376 if (vma->vm_start <= address)
377 goto good_area;
378 if (!(vma->vm_flags & VM_GROWSDOWN))
379 goto bad_area;
380 if (!(fault_code & FAULT_CODE_WRITE)) {
381 /* Non-faulting loads shouldn't expand stack. */
382 insn = get_fault_insn(regs, insn);
383 if ((insn & 0xc0800000) == 0xc0800000) {
384 unsigned char asi;
385
386 if (insn & 0x2000)
387 asi = (regs->tstate >> 24);
388 else
389 asi = (insn >> 5);
390 if ((asi & 0xf2) == 0x82)
391 goto bad_area;
392 }
393 }
394 if (expand_stack(vma, address))
395 goto bad_area;
396 /*
397 * Ok, we have a good vm_area for this memory access, so
398 * we can handle it..
399 */
400good_area:
401 si_code = SEGV_ACCERR;
402
403 /* If we took a ITLB miss on a non-executable page, catch
404 * that here.
405 */
406 if ((fault_code & FAULT_CODE_ITLB) && !(vma->vm_flags & VM_EXEC)) {
407 BUG_ON(address != regs->tpc);
408 BUG_ON(regs->tstate & TSTATE_PRIV);
409 goto bad_area;
410 }
411
412 if (fault_code & FAULT_CODE_WRITE) {
413 if (!(vma->vm_flags & VM_WRITE))
414 goto bad_area;
415
416 /* Spitfire has an icache which does not snoop
417 * processor stores. Later processors do...
418 */
419 if (tlb_type == spitfire &&
420 (vma->vm_flags & VM_EXEC) != 0 &&
421 vma->vm_file != NULL)
422 set_thread_fault_code(fault_code |
423 FAULT_CODE_BLKCOMMIT);
Johannes Weiner759496b2013-09-12 15:13:39 -0700424
425 flags |= FAULT_FLAG_WRITE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 } else {
427 /* Allow reads even for write-only mappings */
428 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
429 goto bad_area;
430 }
431
Kautuk Consul7358e512012-03-26 06:40:49 +0000432 fault = handle_mm_fault(mm, vma, address, flags);
433
434 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
Kirill Tkhai812cb832013-09-14 16:02:11 +0400435 goto exit_exception;
Kautuk Consul7358e512012-03-26 06:40:49 +0000436
Nick Piggin83c54072007-07-19 01:47:05 -0700437 if (unlikely(fault & VM_FAULT_ERROR)) {
438 if (fault & VM_FAULT_OOM)
439 goto out_of_memory;
440 else if (fault & VM_FAULT_SIGBUS)
441 goto do_sigbus;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 BUG();
443 }
Kautuk Consul7358e512012-03-26 06:40:49 +0000444
445 if (flags & FAULT_FLAG_ALLOW_RETRY) {
446 if (fault & VM_FAULT_MAJOR) {
447 current->maj_flt++;
448 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ,
449 1, regs, address);
450 } else {
451 current->min_flt++;
452 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN,
453 1, regs, address);
454 }
455 if (fault & VM_FAULT_RETRY) {
456 flags &= ~FAULT_FLAG_ALLOW_RETRY;
Shaohua Li45cac652012-10-08 16:32:19 -0700457 flags |= FAULT_FLAG_TRIED;
Kautuk Consul7358e512012-03-26 06:40:49 +0000458
459 /* No need to up_read(&mm->mmap_sem) as we would
460 * have already released it in __lock_page_or_retry
461 * in mm/filemap.c.
462 */
463
464 goto retry;
465 }
David S. Millera084b662010-01-20 03:04:14 -0800466 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 up_read(&mm->mmap_sem);
David S. Miller7a1ac522006-03-16 02:02:32 -0800468
469 mm_rss = get_mm_rss(mm);
David Miller9e695d22012-10-08 16:34:29 -0700470#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
David S. Millerdcc1e8d2006-03-22 00:49:59 -0800471 mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE));
472#endif
David S. Miller7bebd832006-03-27 01:07:55 -0800473 if (unlikely(mm_rss >
David S. Millerdcc1e8d2006-03-22 00:49:59 -0800474 mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
475 tsb_grow(mm, MM_TSB_BASE, mm_rss);
David Miller9e695d22012-10-08 16:34:29 -0700476#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
David S. Millerdcc1e8d2006-03-22 00:49:59 -0800477 mm_rss = mm->context.huge_pte_count;
David S. Miller7bebd832006-03-27 01:07:55 -0800478 if (unlikely(mm_rss >
David S. Miller0fbebed2013-02-19 22:34:10 -0800479 mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) {
480 if (mm->context.tsb_block[MM_TSB_HUGE].tsb)
481 tsb_grow(mm, MM_TSB_HUGE, mm_rss);
482 else
483 hugetlb_setup(regs);
484
485 }
David S. Millerdcc1e8d2006-03-22 00:49:59 -0800486#endif
Kirill Tkhai812cb832013-09-14 16:02:11 +0400487exit_exception:
488 exception_exit(prev_state);
David S. Millerefdc1e22005-09-28 21:06:47 -0700489 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490
491 /*
492 * Something tried to access memory that isn't in our memory map..
493 * Fix it, but check if it's kernel or user first..
494 */
495bad_area:
496 insn = get_fault_insn(regs, insn);
497 up_read(&mm->mmap_sem);
498
499handle_kernel_fault:
500 do_kernel_fault(regs, si_code, fault_code, insn, address);
Kirill Tkhai812cb832013-09-14 16:02:11 +0400501 goto exit_exception;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502
503/*
504 * We ran out of memory, or some other thing happened to us that made
505 * us unable to handle the page fault gracefully.
506 */
507out_of_memory:
508 insn = get_fault_insn(regs, insn);
509 up_read(&mm->mmap_sem);
David S. Millera923c282009-08-02 19:17:15 -0700510 if (!(regs->tstate & TSTATE_PRIV)) {
511 pagefault_out_of_memory();
Kirill Tkhai812cb832013-09-14 16:02:11 +0400512 goto exit_exception;
David S. Millera923c282009-08-02 19:17:15 -0700513 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 goto handle_kernel_fault;
515
516intr_or_no_mm:
517 insn = get_fault_insn(regs, 0);
518 goto handle_kernel_fault;
519
520do_sigbus:
521 insn = get_fault_insn(regs, insn);
522 up_read(&mm->mmap_sem);
523
524 /*
525 * Send a sigbus, regardless of whether we were in kernel
526 * or user mode.
527 */
528 do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, insn, fault_code);
529
530 /* Kernel mode? Handle exceptions or die */
531 if (regs->tstate & TSTATE_PRIV)
532 goto handle_kernel_fault;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533}