| /* | |
| * This file contains low-level functions for performing various | |
| * types of TLB invalidations on various processors with no hash | |
| * table. | |
| * | |
| * This file implements the following functions for all no-hash | |
| * processors. Some aren't implemented for some variants. Some | |
| * are inline in tlbflush.h | |
| * | |
| * - tlbil_va | |
| * - tlbil_pid | |
| * - tlbil_all | |
| * - tlbivax_bcast | |
| * | |
| * Code mostly moved over from misc_32.S | |
| * | |
| * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) | |
| * | |
| * Partially rewritten by Cort Dougan (cort@cs.nmt.edu) | |
| * Paul Mackerras, Kumar Gala and Benjamin Herrenschmidt. | |
| * | |
| * This program is free software; you can redistribute it and/or | |
| * modify it under the terms of the GNU General Public License | |
| * as published by the Free Software Foundation; either version | |
| * 2 of the License, or (at your option) any later version. | |
| * | |
| */ | |
| #include <asm/reg.h> | |
| #include <asm/page.h> | |
| #include <asm/cputable.h> | |
| #include <asm/mmu.h> | |
| #include <asm/ppc_asm.h> | |
| #include <asm/asm-offsets.h> | |
| #include <asm/processor.h> | |
| #include <asm/bug.h> | |
| #if defined(CONFIG_40x) | |
| /* | |
| * 40x implementation needs only tlbil_va | |
| */ | |
| _GLOBAL(__tlbil_va) | |
| /* We run the search with interrupts disabled because we have to change | |
| * the PID and I don't want to preempt when that happens. | |
| */ | |
| mfmsr r5 | |
| mfspr r6,SPRN_PID | |
| wrteei 0 | |
| mtspr SPRN_PID,r4 | |
| tlbsx. r3, 0, r3 | |
| mtspr SPRN_PID,r6 | |
| wrtee r5 | |
| bne 1f | |
| sync | |
| /* There are only 64 TLB entries, so r3 < 64, which means bit 25 is | |
| * clear. Since 25 is the V bit in the TLB_TAG, loading this value | |
| * will invalidate the TLB entry. */ | |
| tlbwe r3, r3, TLB_TAG | |
| isync | |
| 1: blr | |
| #elif defined(CONFIG_PPC_8xx) | |
| /* | |
| * Nothing to do for 8xx, everything is inline | |
| */ | |
| #elif defined(CONFIG_44x) /* Includes 47x */ | |
| /* | |
| * 440 implementation uses tlbsx/we for tlbil_va and a full sweep | |
| * of the TLB for everything else. | |
| */ | |
| _GLOBAL(__tlbil_va) | |
| mfspr r5,SPRN_MMUCR | |
| mfmsr r10 | |
| /* | |
| * We write 16 bits of STID since 47x supports that much, we | |
| * will never be passed out of bounds values on 440 (hopefully) | |
| */ | |
| rlwimi r5,r4,0,16,31 | |
| /* We have to run the search with interrupts disabled, otherwise | |
| * an interrupt which causes a TLB miss can clobber the MMUCR | |
| * between the mtspr and the tlbsx. | |
| * | |
| * Critical and Machine Check interrupts take care of saving | |
| * and restoring MMUCR, so only normal interrupts have to be | |
| * taken care of. | |
| */ | |
| wrteei 0 | |
| mtspr SPRN_MMUCR,r5 | |
| tlbsx. r6,0,r3 | |
| bne 10f | |
| sync | |
| BEGIN_MMU_FTR_SECTION | |
| b 2f | |
| END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x) | |
| /* On 440 There are only 64 TLB entries, so r3 < 64, which means bit | |
| * 22, is clear. Since 22 is the V bit in the TLB_PAGEID, loading this | |
| * value will invalidate the TLB entry. | |
| */ | |
| tlbwe r6,r6,PPC44x_TLB_PAGEID | |
| isync | |
| 10: wrtee r10 | |
| blr | |
| 2: | |
| #ifdef CONFIG_PPC_47x | |
| oris r7,r6,0x8000 /* specify way explicitly */ | |
| clrrwi r4,r3,12 /* get an EPN for the hashing with V = 0 */ | |
| ori r4,r4,PPC47x_TLBE_SIZE | |
| tlbwe r4,r7,0 /* write it */ | |
| isync | |
| wrtee r10 | |
| blr | |
| #else /* CONFIG_PPC_47x */ | |
| 1: trap | |
| EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0; | |
| #endif /* !CONFIG_PPC_47x */ | |
| _GLOBAL(_tlbil_all) | |
| _GLOBAL(_tlbil_pid) | |
| BEGIN_MMU_FTR_SECTION | |
| b 2f | |
| END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x) | |
| li r3,0 | |
| sync | |
| /* Load high watermark */ | |
| lis r4,tlb_44x_hwater@ha | |
| lwz r5,tlb_44x_hwater@l(r4) | |
| 1: tlbwe r3,r3,PPC44x_TLB_PAGEID | |
| addi r3,r3,1 | |
| cmpw 0,r3,r5 | |
| ble 1b | |
| isync | |
| blr | |
| 2: | |
| #ifdef CONFIG_PPC_47x | |
| /* 476 variant. There's not simple way to do this, hopefully we'll | |
| * try to limit the amount of such full invalidates | |
| */ | |
| mfmsr r11 /* Interrupts off */ | |
| wrteei 0 | |
| li r3,-1 /* Current set */ | |
| lis r10,tlb_47x_boltmap@h | |
| ori r10,r10,tlb_47x_boltmap@l | |
| lis r7,0x8000 /* Specify way explicitly */ | |
| b 9f /* For each set */ | |
| 1: li r9,4 /* Number of ways */ | |
| li r4,0 /* Current way */ | |
| li r6,0 /* Default entry value 0 */ | |
| andi. r0,r8,1 /* Check if way 0 is bolted */ | |
| mtctr r9 /* Load way counter */ | |
| bne- 3f /* Bolted, skip loading it */ | |
| 2: /* For each way */ | |
| or r5,r3,r4 /* Make way|index for tlbre */ | |
| rlwimi r5,r5,16,8,15 /* Copy index into position */ | |
| tlbre r6,r5,0 /* Read entry */ | |
| 3: addis r4,r4,0x2000 /* Next way */ | |
| andi. r0,r6,PPC47x_TLB0_VALID /* Valid entry ? */ | |
| beq 4f /* Nope, skip it */ | |
| rlwimi r7,r5,0,1,2 /* Insert way number */ | |
| rlwinm r6,r6,0,21,19 /* Clear V */ | |
| tlbwe r6,r7,0 /* Write it */ | |
| 4: bdnz 2b /* Loop for each way */ | |
| srwi r8,r8,1 /* Next boltmap bit */ | |
| 9: cmpwi cr1,r3,255 /* Last set done ? */ | |
| addi r3,r3,1 /* Next set */ | |
| beq cr1,1f /* End of loop */ | |
| andi. r0,r3,0x1f /* Need to load a new boltmap word ? */ | |
| bne 1b /* No, loop */ | |
| lwz r8,0(r10) /* Load boltmap entry */ | |
| addi r10,r10,4 /* Next word */ | |
| b 1b /* Then loop */ | |
| 1: isync /* Sync shadows */ | |
| wrtee r11 | |
| #else /* CONFIG_PPC_47x */ | |
| 1: trap | |
| EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0; | |
| #endif /* !CONFIG_PPC_47x */ | |
| blr | |
| #ifdef CONFIG_PPC_47x | |
| /* | |
| * _tlbivax_bcast is only on 47x. We don't bother doing a runtime | |
| * check though, it will blow up soon enough if we mistakenly try | |
| * to use it on a 440. | |
| */ | |
| _GLOBAL(_tlbivax_bcast) | |
| mfspr r5,SPRN_MMUCR | |
| mfmsr r10 | |
| rlwimi r5,r4,0,16,31 | |
| wrteei 0 | |
| mtspr SPRN_MMUCR,r5 | |
| isync | |
| PPC_TLBIVAX(0, R3) | |
| isync | |
| eieio | |
| tlbsync | |
| BEGIN_FTR_SECTION | |
| b 1f | |
| END_FTR_SECTION_IFSET(CPU_FTR_476_DD2) | |
| sync | |
| wrtee r10 | |
| blr | |
| /* | |
| * DD2 HW could hang if in instruction fetch happens before msync completes. | |
| * Touch enough instruction cache lines to ensure cache hits | |
| */ | |
| 1: mflr r9 | |
| bl 2f | |
| 2: mflr r6 | |
| li r7,32 | |
| PPC_ICBT(0,R6,R7) /* touch next cache line */ | |
| add r6,r6,r7 | |
| PPC_ICBT(0,R6,R7) /* touch next cache line */ | |
| add r6,r6,r7 | |
| PPC_ICBT(0,R6,R7) /* touch next cache line */ | |
| sync | |
| nop | |
| nop | |
| nop | |
| nop | |
| nop | |
| nop | |
| nop | |
| nop | |
| mtlr r9 | |
| wrtee r10 | |
| blr | |
| #endif /* CONFIG_PPC_47x */ | |
| #elif defined(CONFIG_FSL_BOOKE) | |
| /* | |
| * FSL BookE implementations. | |
| * | |
| * Since feature sections are using _SECTION_ELSE we need | |
| * to have the larger code path before the _SECTION_ELSE | |
| */ | |
| /* | |
| * Flush MMU TLB on the local processor | |
| */ | |
| _GLOBAL(_tlbil_all) | |
| BEGIN_MMU_FTR_SECTION | |
| li r3,(MMUCSR0_TLBFI)@l | |
| mtspr SPRN_MMUCSR0, r3 | |
| 1: | |
| mfspr r3,SPRN_MMUCSR0 | |
| andi. r3,r3,MMUCSR0_TLBFI@l | |
| bne 1b | |
| MMU_FTR_SECTION_ELSE | |
| PPC_TLBILX_ALL(0,R0) | |
| ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX) | |
| msync | |
| isync | |
| blr | |
| _GLOBAL(_tlbil_pid) | |
| BEGIN_MMU_FTR_SECTION | |
| slwi r3,r3,16 | |
| mfmsr r10 | |
| wrteei 0 | |
| mfspr r4,SPRN_MAS6 /* save MAS6 */ | |
| mtspr SPRN_MAS6,r3 | |
| PPC_TLBILX_PID(0,R0) | |
| mtspr SPRN_MAS6,r4 /* restore MAS6 */ | |
| wrtee r10 | |
| MMU_FTR_SECTION_ELSE | |
| li r3,(MMUCSR0_TLBFI)@l | |
| mtspr SPRN_MMUCSR0, r3 | |
| 1: | |
| mfspr r3,SPRN_MMUCSR0 | |
| andi. r3,r3,MMUCSR0_TLBFI@l | |
| bne 1b | |
| ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBILX) | |
| msync | |
| isync | |
| blr | |
| /* | |
| * Flush MMU TLB for a particular address, but only on the local processor | |
| * (no broadcast) | |
| */ | |
| _GLOBAL(__tlbil_va) | |
| mfmsr r10 | |
| wrteei 0 | |
| slwi r4,r4,16 | |
| ori r4,r4,(MAS6_ISIZE(BOOK3E_PAGESZ_4K))@l | |
| mtspr SPRN_MAS6,r4 /* assume AS=0 for now */ | |
| BEGIN_MMU_FTR_SECTION | |
| tlbsx 0,r3 | |
| mfspr r4,SPRN_MAS1 /* check valid */ | |
| andis. r3,r4,MAS1_VALID@h | |
| beq 1f | |
| rlwinm r4,r4,0,1,31 | |
| mtspr SPRN_MAS1,r4 | |
| tlbwe | |
| MMU_FTR_SECTION_ELSE | |
| PPC_TLBILX_VA(0,R3) | |
| ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX) | |
| msync | |
| isync | |
| 1: wrtee r10 | |
| blr | |
| #elif defined(CONFIG_PPC_BOOK3E) | |
| /* | |
| * New Book3E (>= 2.06) implementation | |
| * | |
| * Note: We may be able to get away without the interrupt masking stuff | |
| * if we save/restore MAS6 on exceptions that might modify it | |
| */ | |
| _GLOBAL(_tlbil_pid) | |
| slwi r4,r3,MAS6_SPID_SHIFT | |
| mfmsr r10 | |
| wrteei 0 | |
| mtspr SPRN_MAS6,r4 | |
| PPC_TLBILX_PID(0,R0) | |
| wrtee r10 | |
| msync | |
| isync | |
| blr | |
| _GLOBAL(_tlbil_pid_noind) | |
| slwi r4,r3,MAS6_SPID_SHIFT | |
| mfmsr r10 | |
| ori r4,r4,MAS6_SIND | |
| wrteei 0 | |
| mtspr SPRN_MAS6,r4 | |
| PPC_TLBILX_PID(0,R0) | |
| wrtee r10 | |
| msync | |
| isync | |
| blr | |
| _GLOBAL(_tlbil_all) | |
| PPC_TLBILX_ALL(0,R0) | |
| msync | |
| isync | |
| blr | |
| _GLOBAL(_tlbil_va) | |
| mfmsr r10 | |
| wrteei 0 | |
| cmpwi cr0,r6,0 | |
| slwi r4,r4,MAS6_SPID_SHIFT | |
| rlwimi r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK | |
| beq 1f | |
| rlwimi r4,r6,MAS6_SIND_SHIFT,MAS6_SIND | |
| 1: mtspr SPRN_MAS6,r4 /* assume AS=0 for now */ | |
| PPC_TLBILX_VA(0,R3) | |
| msync | |
| isync | |
| wrtee r10 | |
| blr | |
| _GLOBAL(_tlbivax_bcast) | |
| mfmsr r10 | |
| wrteei 0 | |
| cmpwi cr0,r6,0 | |
| slwi r4,r4,MAS6_SPID_SHIFT | |
| rlwimi r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK | |
| beq 1f | |
| rlwimi r4,r6,MAS6_SIND_SHIFT,MAS6_SIND | |
| 1: mtspr SPRN_MAS6,r4 /* assume AS=0 for now */ | |
| PPC_TLBIVAX(0,R3) | |
| eieio | |
| tlbsync | |
| sync | |
| wrtee r10 | |
| blr | |
| _GLOBAL(set_context) | |
| #ifdef CONFIG_BDI_SWITCH | |
| /* Context switch the PTE pointer for the Abatron BDI2000. | |
| * The PGDIR is the second parameter. | |
| */ | |
| lis r5, abatron_pteptrs@h | |
| ori r5, r5, abatron_pteptrs@l | |
| stw r4, 0x4(r5) | |
| #endif | |
| mtspr SPRN_PID,r3 | |
| isync /* Force context change */ | |
| blr | |
| #else | |
| #error Unsupported processor type ! | |
| #endif | |
| #if defined(CONFIG_PPC_FSL_BOOK3E) | |
| /* | |
| * extern void loadcam_entry(unsigned int index) | |
| * | |
| * Load TLBCAM[index] entry in to the L2 CAM MMU | |
| * Must preserve r7, r8, r9, and r10 | |
| */ | |
| _GLOBAL(loadcam_entry) | |
| mflr r5 | |
| LOAD_REG_ADDR_PIC(r4, TLBCAM) | |
| mtlr r5 | |
| mulli r5,r3,TLBCAM_SIZE | |
| add r3,r5,r4 | |
| lwz r4,TLBCAM_MAS0(r3) | |
| mtspr SPRN_MAS0,r4 | |
| lwz r4,TLBCAM_MAS1(r3) | |
| mtspr SPRN_MAS1,r4 | |
| PPC_LL r4,TLBCAM_MAS2(r3) | |
| mtspr SPRN_MAS2,r4 | |
| lwz r4,TLBCAM_MAS3(r3) | |
| mtspr SPRN_MAS3,r4 | |
| BEGIN_MMU_FTR_SECTION | |
| lwz r4,TLBCAM_MAS7(r3) | |
| mtspr SPRN_MAS7,r4 | |
| END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS) | |
| isync | |
| tlbwe | |
| isync | |
| blr | |
| /* | |
| * Load multiple TLB entries at once, using an alternate-space | |
| * trampoline so that we don't have to care about whether the same | |
| * TLB entry maps us before and after. | |
| * | |
| * r3 = first entry to write | |
| * r4 = number of entries to write | |
| * r5 = temporary tlb entry | |
| */ | |
| _GLOBAL(loadcam_multi) | |
| mflr r8 | |
| /* | |
| * Set up temporary TLB entry that is the same as what we're | |
| * running from, but in AS=1. | |
| */ | |
| bl 1f | |
| 1: mflr r6 | |
| tlbsx 0,r8 | |
| mfspr r6,SPRN_MAS1 | |
| ori r6,r6,MAS1_TS | |
| mtspr SPRN_MAS1,r6 | |
| mfspr r6,SPRN_MAS0 | |
| rlwimi r6,r5,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK | |
| mr r7,r5 | |
| mtspr SPRN_MAS0,r6 | |
| isync | |
| tlbwe | |
| isync | |
| /* Switch to AS=1 */ | |
| mfmsr r6 | |
| ori r6,r6,MSR_IS|MSR_DS | |
| mtmsr r6 | |
| isync | |
| mr r9,r3 | |
| add r10,r3,r4 | |
| 2: bl loadcam_entry | |
| addi r9,r9,1 | |
| cmpw r9,r10 | |
| mr r3,r9 | |
| blt 2b | |
| /* Return to AS=0 and clear the temporary entry */ | |
| mfmsr r6 | |
| rlwinm. r6,r6,0,~(MSR_IS|MSR_DS) | |
| mtmsr r6 | |
| isync | |
| li r6,0 | |
| mtspr SPRN_MAS1,r6 | |
| rlwinm r6,r7,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK | |
| oris r6,r6,MAS0_TLBSEL(1)@h | |
| mtspr SPRN_MAS0,r6 | |
| isync | |
| tlbwe | |
| isync | |
| mtlr r8 | |
| blr | |
| #endif |