Adrian Bunk | b00dc83 | 2008-05-19 16:52:27 -0700 | [diff] [blame] | 1 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | * arch/sparc64/mm/fault.c: Page fault handlers for the 64-bit Sparc. |
| 3 | * |
David S. Miller | 4fe3ebe | 2008-07-17 22:11:32 -0700 | [diff] [blame] | 4 | * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 5 | * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) |
| 6 | */ |
| 7 | |
| 8 | #include <asm/head.h> |
| 9 | |
| 10 | #include <linux/string.h> |
| 11 | #include <linux/types.h> |
| 12 | #include <linux/sched.h> |
| 13 | #include <linux/ptrace.h> |
| 14 | #include <linux/mman.h> |
| 15 | #include <linux/signal.h> |
| 16 | #include <linux/mm.h> |
| 17 | #include <linux/module.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | #include <linux/init.h> |
David S. Miller | a084b66 | 2010-01-20 03:04:14 -0800 | [diff] [blame] | 19 | #include <linux/perf_event.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | #include <linux/interrupt.h> |
Prasanna S Panchamukhi | 05e14cb | 2005-09-06 15:19:30 -0700 | [diff] [blame] | 21 | #include <linux/kprobes.h> |
Christoph Hellwig | 1eeb66a | 2007-05-08 00:27:03 -0700 | [diff] [blame] | 22 | #include <linux/kdebug.h> |
David S. Miller | eeabac7 | 2009-02-02 22:08:15 -0800 | [diff] [blame] | 23 | #include <linux/percpu.h> |
Kirill Tkhai | 812cb83 | 2013-09-14 16:02:11 +0400 | [diff] [blame] | 24 | #include <linux/context_tracking.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
| 26 | #include <asm/page.h> |
| 27 | #include <asm/pgtable.h> |
| 28 | #include <asm/openprom.h> |
| 29 | #include <asm/oplib.h> |
| 30 | #include <asm/uaccess.h> |
| 31 | #include <asm/asi.h> |
| 32 | #include <asm/lsu.h> |
| 33 | #include <asm/sections.h> |
David S. Miller | 7a1ac52 | 2006-03-16 02:02:32 -0800 | [diff] [blame] | 34 | #include <asm/mmu_context.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | |
David S. Miller | 4b17764 | 2010-03-01 00:02:23 -0800 | [diff] [blame] | 36 | int show_unhandled_signals = 1; |
| 37 | |
David S. Miller | 4ed5d5e | 2009-12-10 18:08:29 -0800 | [diff] [blame] | 38 | static inline __kprobes int notify_page_fault(struct pt_regs *regs) |
Anil S Keshavamurthy | d98f8f0 | 2006-06-26 00:25:27 -0700 | [diff] [blame] | 39 | { |
David S. Miller | 127cda1 | 2007-05-08 18:25:14 -0700 | [diff] [blame] | 40 | int ret = 0; |
Anil S Keshavamurthy | d98f8f0 | 2006-06-26 00:25:27 -0700 | [diff] [blame] | 41 | |
David S. Miller | 127cda1 | 2007-05-08 18:25:14 -0700 | [diff] [blame] | 42 | /* kprobe_running() needs smp_processor_id() */ |
David S. Miller | 135d082 | 2009-12-10 18:02:19 -0800 | [diff] [blame] | 43 | if (kprobes_built_in() && !user_mode(regs)) { |
David S. Miller | 127cda1 | 2007-05-08 18:25:14 -0700 | [diff] [blame] | 44 | preempt_disable(); |
| 45 | if (kprobe_running() && kprobe_fault_handler(regs, 0)) |
| 46 | ret = 1; |
| 47 | preempt_enable(); |
| 48 | } |
| 49 | return ret; |
Anil S Keshavamurthy | d98f8f0 | 2006-06-26 00:25:27 -0700 | [diff] [blame] | 50 | } |
Anil S Keshavamurthy | d98f8f0 | 2006-06-26 00:25:27 -0700 | [diff] [blame] | 51 | |
Prasanna S Panchamukhi | 05e14cb | 2005-09-06 15:19:30 -0700 | [diff] [blame] | 52 | static void __kprobes unhandled_fault(unsigned long address, |
| 53 | struct task_struct *tsk, |
| 54 | struct pt_regs *regs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 55 | { |
| 56 | if ((unsigned long) address < PAGE_SIZE) { |
| 57 | printk(KERN_ALERT "Unable to handle kernel NULL " |
| 58 | "pointer dereference\n"); |
| 59 | } else { |
| 60 | printk(KERN_ALERT "Unable to handle kernel paging request " |
| 61 | "at virtual address %016lx\n", (unsigned long)address); |
| 62 | } |
| 63 | printk(KERN_ALERT "tsk->{mm,active_mm}->context = %016lx\n", |
| 64 | (tsk->mm ? |
| 65 | CTX_HWBITS(tsk->mm->context) : |
| 66 | CTX_HWBITS(tsk->active_mm->context))); |
| 67 | printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %016lx\n", |
| 68 | (tsk->mm ? (unsigned long) tsk->mm->pgd : |
| 69 | (unsigned long) tsk->active_mm->pgd)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | die_if_kernel("Oops", regs); |
| 71 | } |
| 72 | |
David S. Miller | 4ed5d5e | 2009-12-10 18:08:29 -0800 | [diff] [blame] | 73 | static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 74 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 75 | printk(KERN_CRIT "OOPS: Bogus kernel PC [%016lx] in fault handler\n", |
| 76 | regs->tpc); |
David S. Miller | eb398d1 | 2006-07-22 01:12:09 -0700 | [diff] [blame] | 77 | printk(KERN_CRIT "OOPS: RPC [%016lx]\n", regs->u_regs[15]); |
David S. Miller | 4fe3ebe | 2008-07-17 22:11:32 -0700 | [diff] [blame] | 78 | printk("OOPS: RPC <%pS>\n", (void *) regs->u_regs[15]); |
David S. Miller | bf941d6 | 2006-02-13 18:07:45 -0800 | [diff] [blame] | 79 | printk(KERN_CRIT "OOPS: Fault was to vaddr[%lx]\n", vaddr); |
David S. Miller | c1f193a | 2007-07-30 00:17:12 -0700 | [diff] [blame] | 80 | dump_stack(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 81 | unhandled_fault(regs->tpc, current, regs); |
| 82 | } |
| 83 | |
| 84 | /* |
| 85 | * We now make sure that mmap_sem is held in all paths that call |
| 86 | * this. Additionally, to prevent kswapd from ripping ptes from |
| 87 | * under us, raise interrupts around the time that we look at the |
| 88 | * pte, kswapd will have to wait to get his smp ipi response from |
Hugh Dickins | da16054 | 2005-11-08 10:00:55 -0800 | [diff] [blame] | 89 | * us. vmtruncate likewise. This saves us having to get pte lock. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 90 | */ |
| 91 | static unsigned int get_user_insn(unsigned long tpc) |
| 92 | { |
| 93 | pgd_t *pgdp = pgd_offset(current->mm, tpc); |
| 94 | pud_t *pudp; |
| 95 | pmd_t *pmdp; |
| 96 | pte_t *ptep, pte; |
| 97 | unsigned long pa; |
| 98 | u32 insn = 0; |
| 99 | unsigned long pstate; |
| 100 | |
| 101 | if (pgd_none(*pgdp)) |
| 102 | goto outret; |
| 103 | pudp = pud_offset(pgdp, tpc); |
| 104 | if (pud_none(*pudp)) |
| 105 | goto outret; |
| 106 | pmdp = pmd_offset(pudp, tpc); |
| 107 | if (pmd_none(*pmdp)) |
| 108 | goto outret; |
| 109 | |
| 110 | /* This disables preemption for us as well. */ |
| 111 | __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); |
| 112 | __asm__ __volatile__("wrpr %0, %1, %%pstate" |
| 113 | : : "r" (pstate), "i" (PSTATE_IE)); |
| 114 | ptep = pte_offset_map(pmdp, tpc); |
| 115 | pte = *ptep; |
| 116 | if (!pte_present(pte)) |
| 117 | goto out; |
| 118 | |
David S. Miller | c4bce90 | 2006-02-11 21:57:54 -0800 | [diff] [blame] | 119 | pa = (pte_pfn(pte) << PAGE_SHIFT); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 120 | pa += (tpc & ~PAGE_MASK); |
| 121 | |
| 122 | /* Use phys bypass so we don't pollute dtlb/dcache. */ |
| 123 | __asm__ __volatile__("lduwa [%1] %2, %0" |
| 124 | : "=r" (insn) |
| 125 | : "r" (pa), "i" (ASI_PHYS_USE_EC)); |
| 126 | |
| 127 | out: |
| 128 | pte_unmap(ptep); |
| 129 | __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate)); |
| 130 | outret: |
| 131 | return insn; |
| 132 | } |
| 133 | |
David S. Miller | 4b17764 | 2010-03-01 00:02:23 -0800 | [diff] [blame] | 134 | static inline void |
| 135 | show_signal_msg(struct pt_regs *regs, int sig, int code, |
| 136 | unsigned long address, struct task_struct *tsk) |
| 137 | { |
| 138 | if (!unhandled_signal(tsk, sig)) |
| 139 | return; |
| 140 | |
| 141 | if (!printk_ratelimit()) |
| 142 | return; |
| 143 | |
| 144 | printk("%s%s[%d]: segfault at %lx ip %p (rpc %p) sp %p error %x", |
| 145 | task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG, |
| 146 | tsk->comm, task_pid_nr(tsk), address, |
| 147 | (void *)regs->tpc, (void *)regs->u_regs[UREG_I7], |
| 148 | (void *)regs->u_regs[UREG_FP], code); |
| 149 | |
| 150 | print_vma_addr(KERN_CONT " in ", regs->tpc); |
| 151 | |
| 152 | printk(KERN_CONT "\n"); |
| 153 | } |
| 154 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | static void do_fault_siginfo(int code, int sig, struct pt_regs *regs, |
| 156 | unsigned int insn, int fault_code) |
| 157 | { |
David S. Miller | 4b17764 | 2010-03-01 00:02:23 -0800 | [diff] [blame] | 158 | unsigned long addr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 159 | siginfo_t info; |
| 160 | |
| 161 | info.si_code = code; |
| 162 | info.si_signo = sig; |
| 163 | info.si_errno = 0; |
| 164 | if (fault_code & FAULT_CODE_ITLB) |
David S. Miller | 4b17764 | 2010-03-01 00:02:23 -0800 | [diff] [blame] | 165 | addr = regs->tpc; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | else |
David S. Miller | 4b17764 | 2010-03-01 00:02:23 -0800 | [diff] [blame] | 167 | addr = compute_effective_address(regs, insn, 0); |
| 168 | info.si_addr = (void __user *) addr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 169 | info.si_trapno = 0; |
David S. Miller | 4b17764 | 2010-03-01 00:02:23 -0800 | [diff] [blame] | 170 | |
| 171 | if (unlikely(show_unhandled_signals)) |
| 172 | show_signal_msg(regs, sig, code, addr, current); |
| 173 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 174 | force_sig_info(sig, &info, current); |
| 175 | } |
| 176 | |
| 177 | extern int handle_ldf_stq(u32, struct pt_regs *); |
| 178 | extern int handle_ld_nf(u32, struct pt_regs *); |
| 179 | |
| 180 | static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn) |
| 181 | { |
| 182 | if (!insn) { |
| 183 | if (!regs->tpc || (regs->tpc & 0x3)) |
| 184 | return 0; |
| 185 | if (regs->tstate & TSTATE_PRIV) { |
| 186 | insn = *(unsigned int *) regs->tpc; |
| 187 | } else { |
| 188 | insn = get_user_insn(regs->tpc); |
| 189 | } |
| 190 | } |
| 191 | return insn; |
| 192 | } |
| 193 | |
David S. Miller | 4ed5d5e | 2009-12-10 18:08:29 -0800 | [diff] [blame] | 194 | static void __kprobes do_kernel_fault(struct pt_regs *regs, int si_code, |
| 195 | int fault_code, unsigned int insn, |
| 196 | unsigned long address) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 198 | unsigned char asi = ASI_P; |
| 199 | |
| 200 | if ((!insn) && (regs->tstate & TSTATE_PRIV)) |
| 201 | goto cannot_handle; |
| 202 | |
| 203 | /* If user insn could be read (thus insn is zero), that |
| 204 | * is fine. We will just gun down the process with a signal |
| 205 | * in that case. |
| 206 | */ |
| 207 | |
| 208 | if (!(fault_code & (FAULT_CODE_WRITE|FAULT_CODE_ITLB)) && |
| 209 | (insn & 0xc0800000) == 0xc0800000) { |
| 210 | if (insn & 0x2000) |
| 211 | asi = (regs->tstate >> 24); |
| 212 | else |
| 213 | asi = (insn >> 5); |
| 214 | if ((asi & 0xf2) == 0x82) { |
| 215 | if (insn & 0x1000000) { |
| 216 | handle_ldf_stq(insn, regs); |
| 217 | } else { |
| 218 | /* This was a non-faulting load. Just clear the |
| 219 | * destination register(s) and continue with the next |
| 220 | * instruction. -jj |
| 221 | */ |
| 222 | handle_ld_nf(insn, regs); |
| 223 | } |
| 224 | return; |
| 225 | } |
| 226 | } |
| 227 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 228 | /* Is this in ex_table? */ |
| 229 | if (regs->tstate & TSTATE_PRIV) { |
David S. Miller | 8cf14af | 2005-09-28 20:21:11 -0700 | [diff] [blame] | 230 | const struct exception_table_entry *entry; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 231 | |
David S. Miller | 622eaec | 2008-02-26 17:30:02 -0800 | [diff] [blame] | 232 | entry = search_exception_tables(regs->tpc); |
| 233 | if (entry) { |
David S. Miller | 8cf14af | 2005-09-28 20:21:11 -0700 | [diff] [blame] | 234 | regs->tpc = entry->fixup; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 235 | regs->tnpc = regs->tpc + 4; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 236 | return; |
| 237 | } |
| 238 | } else { |
| 239 | /* The si_code was set to make clear whether |
| 240 | * this was a SEGV_MAPERR or SEGV_ACCERR fault. |
| 241 | */ |
| 242 | do_fault_siginfo(si_code, SIGSEGV, regs, insn, fault_code); |
| 243 | return; |
| 244 | } |
| 245 | |
| 246 | cannot_handle: |
| 247 | unhandled_fault (address, current, regs); |
| 248 | } |
| 249 | |
David S. Miller | 4ed5d5e | 2009-12-10 18:08:29 -0800 | [diff] [blame] | 250 | static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs) |
David S. Miller | 9b02605 | 2009-02-03 16:28:23 -0800 | [diff] [blame] | 251 | { |
| 252 | static int times; |
| 253 | |
| 254 | if (times++ < 10) |
| 255 | printk(KERN_ERR "FAULT[%s:%d]: 32-bit process reports " |
| 256 | "64-bit TPC [%lx]\n", |
| 257 | current->comm, current->pid, |
| 258 | regs->tpc); |
| 259 | show_regs(regs); |
| 260 | } |
| 261 | |
David S. Miller | 4ed5d5e | 2009-12-10 18:08:29 -0800 | [diff] [blame] | 262 | static void noinline __kprobes bogus_32bit_fault_address(struct pt_regs *regs, |
| 263 | unsigned long addr) |
David S. Miller | 9b02605 | 2009-02-03 16:28:23 -0800 | [diff] [blame] | 264 | { |
| 265 | static int times; |
| 266 | |
| 267 | if (times++ < 10) |
| 268 | printk(KERN_ERR "FAULT[%s:%d]: 32-bit process " |
| 269 | "reports 64-bit fault address [%lx]\n", |
| 270 | current->comm, current->pid, addr); |
| 271 | show_regs(regs); |
| 272 | } |
| 273 | |
Prasanna S Panchamukhi | 05e14cb | 2005-09-06 15:19:30 -0700 | [diff] [blame] | 274 | asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 275 | { |
Kirill Tkhai | 812cb83 | 2013-09-14 16:02:11 +0400 | [diff] [blame] | 276 | enum ctx_state prev_state = exception_enter(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 277 | struct mm_struct *mm = current->mm; |
| 278 | struct vm_area_struct *vma; |
| 279 | unsigned int insn = 0; |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 280 | int si_code, fault_code, fault; |
David S. Miller | 7a1ac52 | 2006-03-16 02:02:32 -0800 | [diff] [blame] | 281 | unsigned long address, mm_rss; |
Kautuk Consul | 7358e51 | 2012-03-26 06:40:49 +0000 | [diff] [blame] | 282 | unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 283 | |
| 284 | fault_code = get_thread_fault_code(); |
| 285 | |
David S. Miller | 127cda1 | 2007-05-08 18:25:14 -0700 | [diff] [blame] | 286 | if (notify_page_fault(regs)) |
Kirill Tkhai | 812cb83 | 2013-09-14 16:02:11 +0400 | [diff] [blame] | 287 | goto exit_exception; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 288 | |
| 289 | si_code = SEGV_MAPERR; |
| 290 | address = current_thread_info()->fault_address; |
| 291 | |
| 292 | if ((fault_code & FAULT_CODE_ITLB) && |
| 293 | (fault_code & FAULT_CODE_DTLB)) |
| 294 | BUG(); |
| 295 | |
David S. Miller | eeabac7 | 2009-02-02 22:08:15 -0800 | [diff] [blame] | 296 | if (test_thread_flag(TIF_32BIT)) { |
David S. Miller | 9b02605 | 2009-02-03 16:28:23 -0800 | [diff] [blame] | 297 | if (!(regs->tstate & TSTATE_PRIV)) { |
| 298 | if (unlikely((regs->tpc >> 32) != 0)) { |
| 299 | bogus_32bit_fault_tpc(regs); |
| 300 | goto intr_or_no_mm; |
| 301 | } |
| 302 | } |
| 303 | if (unlikely((address >> 32) != 0)) { |
| 304 | bogus_32bit_fault_address(regs, address); |
| 305 | goto intr_or_no_mm; |
| 306 | } |
David S. Miller | eeabac7 | 2009-02-02 22:08:15 -0800 | [diff] [blame] | 307 | } |
| 308 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 309 | if (regs->tstate & TSTATE_PRIV) { |
David S. Miller | 9b02605 | 2009-02-03 16:28:23 -0800 | [diff] [blame] | 310 | unsigned long tpc = regs->tpc; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 311 | |
| 312 | /* Sanity check the PC. */ |
David S. Miller | be71716 | 2008-02-28 20:38:15 -0800 | [diff] [blame] | 313 | if ((tpc >= KERNBASE && tpc < (unsigned long) __init_end) || |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 314 | (tpc >= MODULES_VADDR && tpc < MODULES_END)) { |
| 315 | /* Valid, no problems... */ |
| 316 | } else { |
David S. Miller | bf941d6 | 2006-02-13 18:07:45 -0800 | [diff] [blame] | 317 | bad_kernel_pc(regs, address); |
Kirill Tkhai | 812cb83 | 2013-09-14 16:02:11 +0400 | [diff] [blame] | 318 | goto exit_exception; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 319 | } |
Johannes Weiner | 759496b | 2013-09-12 15:13:39 -0700 | [diff] [blame] | 320 | } else |
| 321 | flags |= FAULT_FLAG_USER; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 322 | |
| 323 | /* |
| 324 | * If we're in an interrupt or have no user |
| 325 | * context, we must not take the fault.. |
| 326 | */ |
| 327 | if (in_atomic() || !mm) |
| 328 | goto intr_or_no_mm; |
| 329 | |
Peter Zijlstra | a8b0ca1 | 2011-06-27 14:41:57 +0200 | [diff] [blame] | 330 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
David S. Miller | a084b66 | 2010-01-20 03:04:14 -0800 | [diff] [blame] | 331 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 332 | if (!down_read_trylock(&mm->mmap_sem)) { |
| 333 | if ((regs->tstate & TSTATE_PRIV) && |
| 334 | !search_exception_tables(regs->tpc)) { |
| 335 | insn = get_fault_insn(regs, insn); |
| 336 | goto handle_kernel_fault; |
| 337 | } |
Kautuk Consul | 7358e51 | 2012-03-26 06:40:49 +0000 | [diff] [blame] | 338 | |
| 339 | retry: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 340 | down_read(&mm->mmap_sem); |
| 341 | } |
| 342 | |
| 343 | vma = find_vma(mm, address); |
| 344 | if (!vma) |
| 345 | goto bad_area; |
| 346 | |
| 347 | /* Pure DTLB misses do not tell us whether the fault causing |
| 348 | * load/store/atomic was a write or not, it only says that there |
| 349 | * was no match. So in such a case we (carefully) read the |
| 350 | * instruction to try and figure this out. It's an optimization |
| 351 | * so it's ok if we can't do this. |
| 352 | * |
| 353 | * Special hack, window spill/fill knows the exact fault type. |
| 354 | */ |
| 355 | if (((fault_code & |
| 356 | (FAULT_CODE_DTLB | FAULT_CODE_WRITE | FAULT_CODE_WINFIXUP)) == FAULT_CODE_DTLB) && |
| 357 | (vma->vm_flags & VM_WRITE) != 0) { |
| 358 | insn = get_fault_insn(regs, 0); |
| 359 | if (!insn) |
| 360 | goto continue_fault; |
David S. Miller | 73c50a2 | 2006-03-28 13:32:24 -0800 | [diff] [blame] | 361 | /* All loads, stores and atomics have bits 30 and 31 both set |
| 362 | * in the instruction. Bit 21 is set in all stores, but we |
| 363 | * have to avoid prefetches which also have bit 21 set. |
| 364 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 365 | if ((insn & 0xc0200000) == 0xc0200000 && |
David S. Miller | 73c50a2 | 2006-03-28 13:32:24 -0800 | [diff] [blame] | 366 | (insn & 0x01780000) != 0x01680000) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 367 | /* Don't bother updating thread struct value, |
| 368 | * because update_mmu_cache only cares which tlb |
| 369 | * the access came from. |
| 370 | */ |
| 371 | fault_code |= FAULT_CODE_WRITE; |
| 372 | } |
| 373 | } |
| 374 | continue_fault: |
| 375 | |
| 376 | if (vma->vm_start <= address) |
| 377 | goto good_area; |
| 378 | if (!(vma->vm_flags & VM_GROWSDOWN)) |
| 379 | goto bad_area; |
| 380 | if (!(fault_code & FAULT_CODE_WRITE)) { |
| 381 | /* Non-faulting loads shouldn't expand stack. */ |
| 382 | insn = get_fault_insn(regs, insn); |
| 383 | if ((insn & 0xc0800000) == 0xc0800000) { |
| 384 | unsigned char asi; |
| 385 | |
| 386 | if (insn & 0x2000) |
| 387 | asi = (regs->tstate >> 24); |
| 388 | else |
| 389 | asi = (insn >> 5); |
| 390 | if ((asi & 0xf2) == 0x82) |
| 391 | goto bad_area; |
| 392 | } |
| 393 | } |
| 394 | if (expand_stack(vma, address)) |
| 395 | goto bad_area; |
| 396 | /* |
| 397 | * Ok, we have a good vm_area for this memory access, so |
| 398 | * we can handle it.. |
| 399 | */ |
| 400 | good_area: |
| 401 | si_code = SEGV_ACCERR; |
| 402 | |
| 403 | /* If we took a ITLB miss on a non-executable page, catch |
| 404 | * that here. |
| 405 | */ |
| 406 | if ((fault_code & FAULT_CODE_ITLB) && !(vma->vm_flags & VM_EXEC)) { |
| 407 | BUG_ON(address != regs->tpc); |
| 408 | BUG_ON(regs->tstate & TSTATE_PRIV); |
| 409 | goto bad_area; |
| 410 | } |
| 411 | |
| 412 | if (fault_code & FAULT_CODE_WRITE) { |
| 413 | if (!(vma->vm_flags & VM_WRITE)) |
| 414 | goto bad_area; |
| 415 | |
| 416 | /* Spitfire has an icache which does not snoop |
| 417 | * processor stores. Later processors do... |
| 418 | */ |
| 419 | if (tlb_type == spitfire && |
| 420 | (vma->vm_flags & VM_EXEC) != 0 && |
| 421 | vma->vm_file != NULL) |
| 422 | set_thread_fault_code(fault_code | |
| 423 | FAULT_CODE_BLKCOMMIT); |
Johannes Weiner | 759496b | 2013-09-12 15:13:39 -0700 | [diff] [blame] | 424 | |
| 425 | flags |= FAULT_FLAG_WRITE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 426 | } else { |
| 427 | /* Allow reads even for write-only mappings */ |
| 428 | if (!(vma->vm_flags & (VM_READ | VM_EXEC))) |
| 429 | goto bad_area; |
| 430 | } |
| 431 | |
Kautuk Consul | 7358e51 | 2012-03-26 06:40:49 +0000 | [diff] [blame] | 432 | fault = handle_mm_fault(mm, vma, address, flags); |
| 433 | |
| 434 | if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
Kirill Tkhai | 812cb83 | 2013-09-14 16:02:11 +0400 | [diff] [blame] | 435 | goto exit_exception; |
Kautuk Consul | 7358e51 | 2012-03-26 06:40:49 +0000 | [diff] [blame] | 436 | |
Nick Piggin | 83c5407 | 2007-07-19 01:47:05 -0700 | [diff] [blame] | 437 | if (unlikely(fault & VM_FAULT_ERROR)) { |
| 438 | if (fault & VM_FAULT_OOM) |
| 439 | goto out_of_memory; |
| 440 | else if (fault & VM_FAULT_SIGBUS) |
| 441 | goto do_sigbus; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 442 | BUG(); |
| 443 | } |
Kautuk Consul | 7358e51 | 2012-03-26 06:40:49 +0000 | [diff] [blame] | 444 | |
| 445 | if (flags & FAULT_FLAG_ALLOW_RETRY) { |
| 446 | if (fault & VM_FAULT_MAJOR) { |
| 447 | current->maj_flt++; |
| 448 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, |
| 449 | 1, regs, address); |
| 450 | } else { |
| 451 | current->min_flt++; |
| 452 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, |
| 453 | 1, regs, address); |
| 454 | } |
| 455 | if (fault & VM_FAULT_RETRY) { |
| 456 | flags &= ~FAULT_FLAG_ALLOW_RETRY; |
Shaohua Li | 45cac65 | 2012-10-08 16:32:19 -0700 | [diff] [blame] | 457 | flags |= FAULT_FLAG_TRIED; |
Kautuk Consul | 7358e51 | 2012-03-26 06:40:49 +0000 | [diff] [blame] | 458 | |
| 459 | /* No need to up_read(&mm->mmap_sem) as we would |
| 460 | * have already released it in __lock_page_or_retry |
| 461 | * in mm/filemap.c. |
| 462 | */ |
| 463 | |
| 464 | goto retry; |
| 465 | } |
David S. Miller | a084b66 | 2010-01-20 03:04:14 -0800 | [diff] [blame] | 466 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 467 | up_read(&mm->mmap_sem); |
David S. Miller | 7a1ac52 | 2006-03-16 02:02:32 -0800 | [diff] [blame] | 468 | |
| 469 | mm_rss = get_mm_rss(mm); |
David Miller | 9e695d2 | 2012-10-08 16:34:29 -0700 | [diff] [blame] | 470 | #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) |
David S. Miller | dcc1e8d | 2006-03-22 00:49:59 -0800 | [diff] [blame] | 471 | mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE)); |
| 472 | #endif |
David S. Miller | 7bebd83 | 2006-03-27 01:07:55 -0800 | [diff] [blame] | 473 | if (unlikely(mm_rss > |
David S. Miller | dcc1e8d | 2006-03-22 00:49:59 -0800 | [diff] [blame] | 474 | mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit)) |
| 475 | tsb_grow(mm, MM_TSB_BASE, mm_rss); |
David Miller | 9e695d2 | 2012-10-08 16:34:29 -0700 | [diff] [blame] | 476 | #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) |
David S. Miller | dcc1e8d | 2006-03-22 00:49:59 -0800 | [diff] [blame] | 477 | mm_rss = mm->context.huge_pte_count; |
David S. Miller | 7bebd83 | 2006-03-27 01:07:55 -0800 | [diff] [blame] | 478 | if (unlikely(mm_rss > |
David S. Miller | 0fbebed | 2013-02-19 22:34:10 -0800 | [diff] [blame] | 479 | mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) { |
| 480 | if (mm->context.tsb_block[MM_TSB_HUGE].tsb) |
| 481 | tsb_grow(mm, MM_TSB_HUGE, mm_rss); |
| 482 | else |
| 483 | hugetlb_setup(regs); |
| 484 | |
| 485 | } |
David S. Miller | dcc1e8d | 2006-03-22 00:49:59 -0800 | [diff] [blame] | 486 | #endif |
Kirill Tkhai | 812cb83 | 2013-09-14 16:02:11 +0400 | [diff] [blame] | 487 | exit_exception: |
| 488 | exception_exit(prev_state); |
David S. Miller | efdc1e2 | 2005-09-28 21:06:47 -0700 | [diff] [blame] | 489 | return; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 490 | |
| 491 | /* |
| 492 | * Something tried to access memory that isn't in our memory map.. |
| 493 | * Fix it, but check if it's kernel or user first.. |
| 494 | */ |
| 495 | bad_area: |
| 496 | insn = get_fault_insn(regs, insn); |
| 497 | up_read(&mm->mmap_sem); |
| 498 | |
| 499 | handle_kernel_fault: |
| 500 | do_kernel_fault(regs, si_code, fault_code, insn, address); |
Kirill Tkhai | 812cb83 | 2013-09-14 16:02:11 +0400 | [diff] [blame] | 501 | goto exit_exception; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 502 | |
| 503 | /* |
| 504 | * We ran out of memory, or some other thing happened to us that made |
| 505 | * us unable to handle the page fault gracefully. |
| 506 | */ |
| 507 | out_of_memory: |
| 508 | insn = get_fault_insn(regs, insn); |
| 509 | up_read(&mm->mmap_sem); |
David S. Miller | a923c28 | 2009-08-02 19:17:15 -0700 | [diff] [blame] | 510 | if (!(regs->tstate & TSTATE_PRIV)) { |
| 511 | pagefault_out_of_memory(); |
Kirill Tkhai | 812cb83 | 2013-09-14 16:02:11 +0400 | [diff] [blame] | 512 | goto exit_exception; |
David S. Miller | a923c28 | 2009-08-02 19:17:15 -0700 | [diff] [blame] | 513 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 514 | goto handle_kernel_fault; |
| 515 | |
| 516 | intr_or_no_mm: |
| 517 | insn = get_fault_insn(regs, 0); |
| 518 | goto handle_kernel_fault; |
| 519 | |
| 520 | do_sigbus: |
| 521 | insn = get_fault_insn(regs, insn); |
| 522 | up_read(&mm->mmap_sem); |
| 523 | |
| 524 | /* |
| 525 | * Send a sigbus, regardless of whether we were in kernel |
| 526 | * or user mode. |
| 527 | */ |
| 528 | do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, insn, fault_code); |
| 529 | |
| 530 | /* Kernel mode? Handle exceptions or die */ |
| 531 | if (regs->tstate & TSTATE_PRIV) |
| 532 | goto handle_kernel_fault; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 533 | } |