Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * arch/ppc64/kernel/u3_iommu.c |
| 3 | * |
| 4 | * Copyright (C) 2004 Olof Johansson <olof@austin.ibm.com>, IBM Corporation |
| 5 | * |
| 6 | * Based on pSeries_iommu.c: |
| 7 | * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation |
| 8 | * Copyright (C) 2004 Olof Johansson <olof@austin.ibm.com>, IBM Corporation |
| 9 | * |
| 10 | * Dynamic DMA mapping support, Apple U3 & IBM CPC925 "DART" iommu. |
| 11 | * |
| 12 | * |
| 13 | * This program is free software; you can redistribute it and/or modify |
| 14 | * it under the terms of the GNU General Public License as published by |
| 15 | * the Free Software Foundation; either version 2 of the License, or |
| 16 | * (at your option) any later version. |
| 17 | * |
| 18 | * This program is distributed in the hope that it will be useful, |
| 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 20 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 21 | * GNU General Public License for more details. |
| 22 | * |
| 23 | * You should have received a copy of the GNU General Public License |
| 24 | * along with this program; if not, write to the Free Software |
| 25 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
| 26 | */ |
| 27 | |
| 28 | #include <linux/config.h> |
| 29 | #include <linux/init.h> |
| 30 | #include <linux/types.h> |
| 31 | #include <linux/slab.h> |
| 32 | #include <linux/mm.h> |
| 33 | #include <linux/spinlock.h> |
| 34 | #include <linux/string.h> |
| 35 | #include <linux/pci.h> |
| 36 | #include <linux/dma-mapping.h> |
| 37 | #include <linux/vmalloc.h> |
| 38 | #include <asm/io.h> |
| 39 | #include <asm/prom.h> |
| 40 | #include <asm/ppcdebug.h> |
| 41 | #include <asm/iommu.h> |
| 42 | #include <asm/pci-bridge.h> |
| 43 | #include <asm/machdep.h> |
| 44 | #include <asm/abs_addr.h> |
| 45 | #include <asm/cacheflush.h> |
| 46 | #include <asm/lmb.h> |
Olof Johansson | c707ffc | 2005-09-20 13:45:41 +1000 | [diff] [blame] | 47 | #include <asm/dart.h> |
Stephen Rothwell | d387899 | 2005-09-28 02:50:25 +1000 | [diff] [blame^] | 48 | #include <asm/ppc-pci.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 49 | |
| 50 | extern int iommu_force_on; |
| 51 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 52 | /* Physical base address and size of the DART table */ |
| 53 | unsigned long dart_tablebase; /* exported to htab_initialize */ |
| 54 | static unsigned long dart_tablesize; |
| 55 | |
| 56 | /* Virtual base address of the DART table */ |
| 57 | static u32 *dart_vbase; |
| 58 | |
| 59 | /* Mapped base address for the dart */ |
| 60 | static unsigned int *dart; |
| 61 | |
| 62 | /* Dummy val that entries are set to when unused */ |
| 63 | static unsigned int dart_emptyval; |
| 64 | |
| 65 | static struct iommu_table iommu_table_u3; |
| 66 | static int iommu_table_u3_inited; |
| 67 | static int dart_dirty; |
| 68 | |
| 69 | #define DBG(...) |
| 70 | |
| 71 | static inline void dart_tlb_invalidate_all(void) |
| 72 | { |
| 73 | unsigned long l = 0; |
| 74 | unsigned int reg; |
| 75 | unsigned long limit; |
| 76 | |
| 77 | DBG("dart: flush\n"); |
| 78 | |
| 79 | /* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the |
| 80 | * control register and wait for it to clear. |
| 81 | * |
| 82 | * Gotcha: Sometimes, the DART won't detect that the bit gets |
| 83 | * set. If so, clear it and set it again. |
| 84 | */ |
| 85 | |
| 86 | limit = 0; |
| 87 | |
| 88 | retry: |
| 89 | reg = in_be32((unsigned int *)dart+DARTCNTL); |
| 90 | reg |= DARTCNTL_FLUSHTLB; |
| 91 | out_be32((unsigned int *)dart+DARTCNTL, reg); |
| 92 | |
| 93 | l = 0; |
| 94 | while ((in_be32((unsigned int *)dart+DARTCNTL) & DARTCNTL_FLUSHTLB) && |
| 95 | l < (1L<<limit)) { |
| 96 | l++; |
| 97 | } |
| 98 | if (l == (1L<<limit)) { |
| 99 | if (limit < 4) { |
| 100 | limit++; |
| 101 | reg = in_be32((unsigned int *)dart+DARTCNTL); |
| 102 | reg &= ~DARTCNTL_FLUSHTLB; |
| 103 | out_be32((unsigned int *)dart+DARTCNTL, reg); |
| 104 | goto retry; |
| 105 | } else |
| 106 | panic("U3-DART: TLB did not flush after waiting a long " |
| 107 | "time. Buggy U3 ?"); |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | static void dart_flush(struct iommu_table *tbl) |
| 112 | { |
| 113 | if (dart_dirty) |
| 114 | dart_tlb_invalidate_all(); |
| 115 | dart_dirty = 0; |
| 116 | } |
| 117 | |
| 118 | static void dart_build(struct iommu_table *tbl, long index, |
| 119 | long npages, unsigned long uaddr, |
| 120 | enum dma_data_direction direction) |
| 121 | { |
| 122 | unsigned int *dp; |
| 123 | unsigned int rpn; |
| 124 | |
| 125 | DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr); |
| 126 | |
Olof Johansson | d0035c62 | 2005-09-20 13:46:44 +1000 | [diff] [blame] | 127 | index <<= DART_PAGE_FACTOR; |
| 128 | npages <<= DART_PAGE_FACTOR; |
| 129 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 130 | dp = ((unsigned int*)tbl->it_base) + index; |
| 131 | |
| 132 | /* On U3, all memory is contigous, so we can move this |
| 133 | * out of the loop. |
| 134 | */ |
| 135 | while (npages--) { |
Olof Johansson | d0035c62 | 2005-09-20 13:46:44 +1000 | [diff] [blame] | 136 | rpn = virt_to_abs(uaddr) >> DART_PAGE_SHIFT; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 137 | |
| 138 | *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK); |
| 139 | |
| 140 | rpn++; |
Olof Johansson | d0035c62 | 2005-09-20 13:46:44 +1000 | [diff] [blame] | 141 | uaddr += DART_PAGE_SIZE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 142 | } |
| 143 | |
| 144 | dart_dirty = 1; |
| 145 | } |
| 146 | |
| 147 | |
| 148 | static void dart_free(struct iommu_table *tbl, long index, long npages) |
| 149 | { |
| 150 | unsigned int *dp; |
| 151 | |
| 152 | /* We don't worry about flushing the TLB cache. The only drawback of |
| 153 | * not doing it is that we won't catch buggy device drivers doing |
| 154 | * bad DMAs, but then no 32-bit architecture ever does either. |
| 155 | */ |
| 156 | |
| 157 | DBG("dart: free at: %lx, %lx\n", index, npages); |
| 158 | |
Olof Johansson | d0035c62 | 2005-09-20 13:46:44 +1000 | [diff] [blame] | 159 | index <<= DART_PAGE_FACTOR; |
| 160 | npages <<= DART_PAGE_FACTOR; |
| 161 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 162 | dp = ((unsigned int *)tbl->it_base) + index; |
| 163 | |
| 164 | while (npages--) |
| 165 | *(dp++) = dart_emptyval; |
| 166 | } |
| 167 | |
| 168 | |
| 169 | static int dart_init(struct device_node *dart_node) |
| 170 | { |
| 171 | unsigned int regword; |
| 172 | unsigned int i; |
| 173 | unsigned long tmp; |
| 174 | |
| 175 | if (dart_tablebase == 0 || dart_tablesize == 0) { |
| 176 | printk(KERN_INFO "U3-DART: table not allocated, using direct DMA\n"); |
| 177 | return -ENODEV; |
| 178 | } |
| 179 | |
| 180 | /* Make sure nothing from the DART range remains in the CPU cache |
| 181 | * from a previous mapping that existed before the kernel took |
| 182 | * over |
| 183 | */ |
| 184 | flush_dcache_phys_range(dart_tablebase, dart_tablebase + dart_tablesize); |
| 185 | |
| 186 | /* Allocate a spare page to map all invalid DART pages. We need to do |
| 187 | * that to work around what looks like a problem with the HT bridge |
| 188 | * prefetching into invalid pages and corrupting data |
| 189 | */ |
Olof Johansson | d0035c62 | 2005-09-20 13:46:44 +1000 | [diff] [blame] | 190 | tmp = lmb_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 191 | if (!tmp) |
| 192 | panic("U3-DART: Cannot allocate spare page!"); |
Olof Johansson | d0035c62 | 2005-09-20 13:46:44 +1000 | [diff] [blame] | 193 | dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) & DARTMAP_RPNMASK); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 194 | |
| 195 | /* Map in DART registers. FIXME: Use device node to get base address */ |
| 196 | dart = ioremap(DART_BASE, 0x7000); |
| 197 | if (dart == NULL) |
| 198 | panic("U3-DART: Cannot map registers!"); |
| 199 | |
| 200 | /* Set initial control register contents: table base, |
| 201 | * table size and enable bit |
| 202 | */ |
| 203 | regword = DARTCNTL_ENABLE | |
Olof Johansson | d0035c62 | 2005-09-20 13:46:44 +1000 | [diff] [blame] | 204 | ((dart_tablebase >> DART_PAGE_SHIFT) << DARTCNTL_BASE_SHIFT) | |
| 205 | (((dart_tablesize >> DART_PAGE_SHIFT) & DARTCNTL_SIZE_MASK) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 206 | << DARTCNTL_SIZE_SHIFT); |
| 207 | dart_vbase = ioremap(virt_to_abs(dart_tablebase), dart_tablesize); |
| 208 | |
| 209 | /* Fill initial table */ |
| 210 | for (i = 0; i < dart_tablesize/4; i++) |
| 211 | dart_vbase[i] = dart_emptyval; |
| 212 | |
| 213 | /* Initialize DART with table base and enable it. */ |
| 214 | out_be32((unsigned int *)dart, regword); |
| 215 | |
| 216 | /* Invalidate DART to get rid of possible stale TLBs */ |
| 217 | dart_tlb_invalidate_all(); |
| 218 | |
| 219 | printk(KERN_INFO "U3/CPC925 DART IOMMU initialized\n"); |
| 220 | |
| 221 | return 0; |
| 222 | } |
| 223 | |
| 224 | static void iommu_table_u3_setup(void) |
| 225 | { |
| 226 | iommu_table_u3.it_busno = 0; |
| 227 | iommu_table_u3.it_offset = 0; |
| 228 | /* it_size is in number of entries */ |
| 229 | iommu_table_u3.it_size = dart_tablesize / sizeof(u32); |
| 230 | |
| 231 | /* Initialize the common IOMMU code */ |
| 232 | iommu_table_u3.it_base = (unsigned long)dart_vbase; |
| 233 | iommu_table_u3.it_index = 0; |
| 234 | iommu_table_u3.it_blocksize = 1; |
| 235 | iommu_init_table(&iommu_table_u3); |
| 236 | |
| 237 | /* Reserve the last page of the DART to avoid possible prefetch |
| 238 | * past the DART mapped area |
| 239 | */ |
| 240 | set_bit(iommu_table_u3.it_size - 1, iommu_table_u3.it_map); |
| 241 | } |
| 242 | |
| 243 | static void iommu_dev_setup_u3(struct pci_dev *dev) |
| 244 | { |
| 245 | struct device_node *dn; |
| 246 | |
| 247 | /* We only have one iommu table on the mac for now, which makes |
| 248 | * things simple. Setup all PCI devices to point to this table |
| 249 | * |
| 250 | * We must use pci_device_to_OF_node() to make sure that |
| 251 | * we get the real "final" pointer to the device in the |
| 252 | * pci_dev sysdata and not the temporary PHB one |
| 253 | */ |
| 254 | dn = pci_device_to_OF_node(dev); |
| 255 | |
| 256 | if (dn) |
Paul Mackerras | 1635317 | 2005-09-06 13:17:54 +1000 | [diff] [blame] | 257 | PCI_DN(dn)->iommu_table = &iommu_table_u3; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 258 | } |
| 259 | |
| 260 | static void iommu_bus_setup_u3(struct pci_bus *bus) |
| 261 | { |
| 262 | struct device_node *dn; |
| 263 | |
| 264 | if (!iommu_table_u3_inited) { |
| 265 | iommu_table_u3_inited = 1; |
| 266 | iommu_table_u3_setup(); |
| 267 | } |
| 268 | |
| 269 | dn = pci_bus_to_OF_node(bus); |
| 270 | |
| 271 | if (dn) |
Paul Mackerras | 1635317 | 2005-09-06 13:17:54 +1000 | [diff] [blame] | 272 | PCI_DN(dn)->iommu_table = &iommu_table_u3; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 273 | } |
| 274 | |
| 275 | static void iommu_dev_setup_null(struct pci_dev *dev) { } |
| 276 | static void iommu_bus_setup_null(struct pci_bus *bus) { } |
| 277 | |
| 278 | void iommu_init_early_u3(void) |
| 279 | { |
| 280 | struct device_node *dn; |
| 281 | |
| 282 | /* Find the DART in the device-tree */ |
| 283 | dn = of_find_compatible_node(NULL, "dart", "u3-dart"); |
| 284 | if (dn == NULL) |
| 285 | return; |
| 286 | |
| 287 | /* Setup low level TCE operations for the core IOMMU code */ |
| 288 | ppc_md.tce_build = dart_build; |
| 289 | ppc_md.tce_free = dart_free; |
| 290 | ppc_md.tce_flush = dart_flush; |
| 291 | |
| 292 | /* Initialize the DART HW */ |
| 293 | if (dart_init(dn)) { |
| 294 | /* If init failed, use direct iommu and null setup functions */ |
| 295 | ppc_md.iommu_dev_setup = iommu_dev_setup_null; |
| 296 | ppc_md.iommu_bus_setup = iommu_bus_setup_null; |
| 297 | |
| 298 | /* Setup pci_dma ops */ |
| 299 | pci_direct_iommu_init(); |
| 300 | } else { |
| 301 | ppc_md.iommu_dev_setup = iommu_dev_setup_u3; |
| 302 | ppc_md.iommu_bus_setup = iommu_bus_setup_u3; |
| 303 | |
| 304 | /* Setup pci_dma ops */ |
| 305 | pci_iommu_init(); |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | |
| 310 | void __init alloc_u3_dart_table(void) |
| 311 | { |
| 312 | /* Only reserve DART space if machine has more than 2GB of RAM |
| 313 | * or if requested with iommu=on on cmdline. |
| 314 | */ |
| 315 | if (lmb_end_of_DRAM() <= 0x80000000ull && !iommu_force_on) |
| 316 | return; |
| 317 | |
| 318 | /* 512 pages (2MB) is max DART tablesize. */ |
| 319 | dart_tablesize = 1UL << 21; |
| 320 | /* 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we |
| 321 | * will blow up an entire large page anyway in the kernel mapping |
| 322 | */ |
| 323 | dart_tablebase = (unsigned long) |
| 324 | abs_to_virt(lmb_alloc_base(1UL<<24, 1UL<<24, 0x80000000L)); |
| 325 | |
| 326 | printk(KERN_INFO "U3-DART allocated at: %lx\n", dart_tablebase); |
| 327 | } |