Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar |
| 3 | * Copyright (C) 2005-2006, Thomas Gleixner, Russell King |
| 4 | * |
| 5 | * This file contains the interrupt descriptor management code |
| 6 | * |
| 7 | * Detailed information is available in Documentation/DocBook/genericirq |
| 8 | * |
| 9 | */ |
| 10 | #include <linux/irq.h> |
| 11 | #include <linux/slab.h> |
Paul Gortmaker | ec53cf2 | 2011-09-19 20:33:19 -0400 | [diff] [blame] | 12 | #include <linux/export.h> |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 13 | #include <linux/interrupt.h> |
| 14 | #include <linux/kernel_stat.h> |
| 15 | #include <linux/radix-tree.h> |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 16 | #include <linux/bitmap.h> |
Marc Zyngier | 76ba59f | 2014-08-26 11:03:16 +0100 | [diff] [blame] | 17 | #include <linux/irqdomain.h> |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 18 | |
| 19 | #include "internals.h" |
| 20 | |
| 21 | /* |
| 22 | * lockdep: we want to handle all irq_desc locks as a single lock-class: |
| 23 | */ |
Thomas Gleixner | 78f90d9 | 2010-09-29 17:18:47 +0200 | [diff] [blame] | 24 | static struct lock_class_key irq_desc_lock_class; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 25 | |
Thomas Gleixner | fe05143 | 2011-05-18 12:53:03 +0200 | [diff] [blame] | 26 | #if defined(CONFIG_SMP) |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 27 | static void __init init_irq_default_affinity(void) |
| 28 | { |
| 29 | alloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT); |
| 30 | cpumask_setall(irq_default_affinity); |
| 31 | } |
| 32 | #else |
| 33 | static void __init init_irq_default_affinity(void) |
| 34 | { |
| 35 | } |
| 36 | #endif |
| 37 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 38 | #ifdef CONFIG_SMP |
| 39 | static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) |
| 40 | { |
Jiang Liu | 9df872f | 2015-06-03 11:47:50 +0800 | [diff] [blame] | 41 | if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity, |
| 42 | gfp, node)) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 43 | return -ENOMEM; |
| 44 | |
| 45 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 46 | if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) { |
Jiang Liu | 9df872f | 2015-06-03 11:47:50 +0800 | [diff] [blame] | 47 | free_cpumask_var(desc->irq_common_data.affinity); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 48 | return -ENOMEM; |
| 49 | } |
| 50 | #endif |
| 51 | return 0; |
| 52 | } |
| 53 | |
| 54 | static void desc_smp_init(struct irq_desc *desc, int node) |
| 55 | { |
Jiang Liu | 9df872f | 2015-06-03 11:47:50 +0800 | [diff] [blame] | 56 | cpumask_copy(desc->irq_common_data.affinity, irq_default_affinity); |
Thomas Gleixner | b7b2933 | 2010-09-29 18:46:55 +0200 | [diff] [blame] | 57 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 58 | cpumask_clear(desc->pending_mask); |
| 59 | #endif |
Jiang Liu | 449e9ca | 2015-06-01 16:05:16 +0800 | [diff] [blame] | 60 | #ifdef CONFIG_NUMA |
| 61 | desc->irq_common_data.node = node; |
| 62 | #endif |
Thomas Gleixner | b7b2933 | 2010-09-29 18:46:55 +0200 | [diff] [blame] | 63 | } |
| 64 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 65 | #else |
| 66 | static inline int |
| 67 | alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; } |
| 68 | static inline void desc_smp_init(struct irq_desc *desc, int node) { } |
| 69 | #endif |
| 70 | |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 71 | static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, |
| 72 | struct module *owner) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 73 | { |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 74 | int cpu; |
| 75 | |
Jiang Liu | af7080e | 2015-06-01 16:05:21 +0800 | [diff] [blame] | 76 | desc->irq_common_data.handler_data = NULL; |
Jiang Liu | b237721 | 2015-06-01 16:05:43 +0800 | [diff] [blame] | 77 | desc->irq_common_data.msi_desc = NULL; |
Jiang Liu | af7080e | 2015-06-01 16:05:21 +0800 | [diff] [blame] | 78 | |
Jiang Liu | 0d0b4c8 | 2015-06-01 16:05:12 +0800 | [diff] [blame] | 79 | desc->irq_data.common = &desc->irq_common_data; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 80 | desc->irq_data.irq = irq; |
| 81 | desc->irq_data.chip = &no_irq_chip; |
| 82 | desc->irq_data.chip_data = NULL; |
Thomas Gleixner | f9e4989 | 2011-02-09 14:54:49 +0100 | [diff] [blame] | 83 | irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS); |
Thomas Gleixner | 801a0e9 | 2011-03-27 11:02:49 +0200 | [diff] [blame] | 84 | irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 85 | desc->handle_irq = handle_bad_irq; |
| 86 | desc->depth = 1; |
Thomas Gleixner | b7b2933 | 2010-09-29 18:46:55 +0200 | [diff] [blame] | 87 | desc->irq_count = 0; |
| 88 | desc->irqs_unhandled = 0; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 89 | desc->name = NULL; |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 90 | desc->owner = owner; |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 91 | for_each_possible_cpu(cpu) |
| 92 | *per_cpu_ptr(desc->kstat_irqs, cpu) = 0; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 93 | desc_smp_init(desc, node); |
| 94 | } |
| 95 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 96 | int nr_irqs = NR_IRQS; |
| 97 | EXPORT_SYMBOL_GPL(nr_irqs); |
| 98 | |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 99 | static DEFINE_MUTEX(sparse_irq_lock); |
Thomas Gleixner | c1ee626 | 2011-02-17 17:45:15 +0100 | [diff] [blame] | 100 | static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 101 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 102 | #ifdef CONFIG_SPARSE_IRQ |
| 103 | |
Thomas Gleixner | baa0d23 | 2010-10-05 15:14:35 +0200 | [diff] [blame] | 104 | static RADIX_TREE(irq_desc_tree, GFP_KERNEL); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 105 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 106 | static void irq_insert_desc(unsigned int irq, struct irq_desc *desc) |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 107 | { |
| 108 | radix_tree_insert(&irq_desc_tree, irq, desc); |
| 109 | } |
| 110 | |
| 111 | struct irq_desc *irq_to_desc(unsigned int irq) |
| 112 | { |
| 113 | return radix_tree_lookup(&irq_desc_tree, irq); |
| 114 | } |
Jiri Kosina | 3911ff3 | 2012-05-13 12:13:15 +0200 | [diff] [blame] | 115 | EXPORT_SYMBOL(irq_to_desc); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 116 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 117 | static void delete_irq_desc(unsigned int irq) |
| 118 | { |
| 119 | radix_tree_delete(&irq_desc_tree, irq); |
| 120 | } |
| 121 | |
| 122 | #ifdef CONFIG_SMP |
| 123 | static void free_masks(struct irq_desc *desc) |
| 124 | { |
| 125 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
| 126 | free_cpumask_var(desc->pending_mask); |
| 127 | #endif |
Jiang Liu | 9df872f | 2015-06-03 11:47:50 +0800 | [diff] [blame] | 128 | free_cpumask_var(desc->irq_common_data.affinity); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 129 | } |
| 130 | #else |
| 131 | static inline void free_masks(struct irq_desc *desc) { } |
| 132 | #endif |
| 133 | |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 134 | void irq_lock_sparse(void) |
| 135 | { |
| 136 | mutex_lock(&sparse_irq_lock); |
| 137 | } |
| 138 | |
| 139 | void irq_unlock_sparse(void) |
| 140 | { |
| 141 | mutex_unlock(&sparse_irq_lock); |
| 142 | } |
| 143 | |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 144 | static struct irq_desc *alloc_desc(int irq, int node, struct module *owner) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 145 | { |
| 146 | struct irq_desc *desc; |
Thomas Gleixner | baa0d23 | 2010-10-05 15:14:35 +0200 | [diff] [blame] | 147 | gfp_t gfp = GFP_KERNEL; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 148 | |
| 149 | desc = kzalloc_node(sizeof(*desc), gfp, node); |
| 150 | if (!desc) |
| 151 | return NULL; |
| 152 | /* allocate based on nr_cpu_ids */ |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 153 | desc->kstat_irqs = alloc_percpu(unsigned int); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 154 | if (!desc->kstat_irqs) |
| 155 | goto err_desc; |
| 156 | |
| 157 | if (alloc_masks(desc, gfp, node)) |
| 158 | goto err_kstat; |
| 159 | |
| 160 | raw_spin_lock_init(&desc->lock); |
| 161 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); |
| 162 | |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 163 | desc_set_defaults(irq, desc, node, owner); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 164 | |
| 165 | return desc; |
| 166 | |
| 167 | err_kstat: |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 168 | free_percpu(desc->kstat_irqs); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 169 | err_desc: |
| 170 | kfree(desc); |
| 171 | return NULL; |
| 172 | } |
| 173 | |
| 174 | static void free_desc(unsigned int irq) |
| 175 | { |
| 176 | struct irq_desc *desc = irq_to_desc(irq); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 177 | |
Thomas Gleixner | 13bfe99 | 2010-09-30 02:46:07 +0200 | [diff] [blame] | 178 | unregister_irq_proc(irq, desc); |
| 179 | |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 180 | /* |
| 181 | * sparse_irq_lock protects also show_interrupts() and |
| 182 | * kstat_irq_usr(). Once we deleted the descriptor from the |
| 183 | * sparse tree we can free it. Access in proc will fail to |
| 184 | * lookup the descriptor. |
| 185 | */ |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 186 | mutex_lock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 187 | delete_irq_desc(irq); |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 188 | mutex_unlock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 189 | |
| 190 | free_masks(desc); |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 191 | free_percpu(desc->kstat_irqs); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 192 | kfree(desc); |
| 193 | } |
| 194 | |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 195 | static int alloc_descs(unsigned int start, unsigned int cnt, int node, |
| 196 | struct module *owner) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 197 | { |
| 198 | struct irq_desc *desc; |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 199 | int i; |
| 200 | |
| 201 | for (i = 0; i < cnt; i++) { |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 202 | desc = alloc_desc(start + i, node, owner); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 203 | if (!desc) |
| 204 | goto err; |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 205 | mutex_lock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 206 | irq_insert_desc(start + i, desc); |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 207 | mutex_unlock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 208 | } |
| 209 | return start; |
| 210 | |
| 211 | err: |
| 212 | for (i--; i >= 0; i--) |
| 213 | free_desc(start + i); |
| 214 | |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 215 | mutex_lock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 216 | bitmap_clear(allocated_irqs, start, cnt); |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 217 | mutex_unlock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 218 | return -ENOMEM; |
| 219 | } |
| 220 | |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 221 | static int irq_expand_nr_irqs(unsigned int nr) |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 222 | { |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 223 | if (nr > IRQ_BITMAP_BITS) |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 224 | return -ENOMEM; |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 225 | nr_irqs = nr; |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 226 | return 0; |
| 227 | } |
| 228 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 229 | int __init early_irq_init(void) |
| 230 | { |
Thomas Gleixner | b683de2 | 2010-09-27 20:55:03 +0200 | [diff] [blame] | 231 | int i, initcnt, node = first_online_node; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 232 | struct irq_desc *desc; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 233 | |
| 234 | init_irq_default_affinity(); |
| 235 | |
Thomas Gleixner | b683de2 | 2010-09-27 20:55:03 +0200 | [diff] [blame] | 236 | /* Let arch update nr_irqs and return the nr of preallocated irqs */ |
| 237 | initcnt = arch_probe_nr_irqs(); |
| 238 | printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 239 | |
Thomas Gleixner | c1ee626 | 2011-02-17 17:45:15 +0100 | [diff] [blame] | 240 | if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS)) |
| 241 | nr_irqs = IRQ_BITMAP_BITS; |
| 242 | |
| 243 | if (WARN_ON(initcnt > IRQ_BITMAP_BITS)) |
| 244 | initcnt = IRQ_BITMAP_BITS; |
| 245 | |
| 246 | if (initcnt > nr_irqs) |
| 247 | nr_irqs = initcnt; |
| 248 | |
Thomas Gleixner | b683de2 | 2010-09-27 20:55:03 +0200 | [diff] [blame] | 249 | for (i = 0; i < initcnt; i++) { |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 250 | desc = alloc_desc(i, node, NULL); |
Thomas Gleixner | aa99ec0 | 2010-09-27 20:02:56 +0200 | [diff] [blame] | 251 | set_bit(i, allocated_irqs); |
| 252 | irq_insert_desc(i, desc); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 253 | } |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 254 | return arch_early_irq_init(); |
| 255 | } |
| 256 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 257 | #else /* !CONFIG_SPARSE_IRQ */ |
| 258 | |
| 259 | struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { |
| 260 | [0 ... NR_IRQS-1] = { |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 261 | .handle_irq = handle_bad_irq, |
| 262 | .depth = 1, |
| 263 | .lock = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock), |
| 264 | } |
| 265 | }; |
| 266 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 267 | int __init early_irq_init(void) |
| 268 | { |
Thomas Gleixner | aa99ec0 | 2010-09-27 20:02:56 +0200 | [diff] [blame] | 269 | int count, i, node = first_online_node; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 270 | struct irq_desc *desc; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 271 | |
| 272 | init_irq_default_affinity(); |
| 273 | |
| 274 | printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS); |
| 275 | |
| 276 | desc = irq_desc; |
| 277 | count = ARRAY_SIZE(irq_desc); |
| 278 | |
| 279 | for (i = 0; i < count; i++) { |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 280 | desc[i].kstat_irqs = alloc_percpu(unsigned int); |
Linus Walleij | e7fbad3 | 2011-05-31 18:14:39 +0200 | [diff] [blame] | 281 | alloc_masks(&desc[i], GFP_KERNEL, node); |
| 282 | raw_spin_lock_init(&desc[i].lock); |
Thomas Gleixner | 154cd38 | 2010-09-22 15:58:45 +0200 | [diff] [blame] | 283 | lockdep_set_class(&desc[i].lock, &irq_desc_lock_class); |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 284 | desc_set_defaults(i, &desc[i], node, NULL); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 285 | } |
| 286 | return arch_early_irq_init(); |
| 287 | } |
| 288 | |
| 289 | struct irq_desc *irq_to_desc(unsigned int irq) |
| 290 | { |
| 291 | return (irq < NR_IRQS) ? irq_desc + irq : NULL; |
| 292 | } |
Paul Gortmaker | 2c45aad | 2014-02-10 13:39:53 -0500 | [diff] [blame] | 293 | EXPORT_SYMBOL(irq_to_desc); |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 294 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 295 | static void free_desc(unsigned int irq) |
| 296 | { |
Thomas Gleixner | d8179bc | 2014-05-07 15:44:23 +0000 | [diff] [blame] | 297 | struct irq_desc *desc = irq_to_desc(irq); |
| 298 | unsigned long flags; |
| 299 | |
| 300 | raw_spin_lock_irqsave(&desc->lock, flags); |
Jiang Liu | 6783011 | 2015-06-01 16:05:13 +0800 | [diff] [blame] | 301 | desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL); |
Thomas Gleixner | d8179bc | 2014-05-07 15:44:23 +0000 | [diff] [blame] | 302 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 303 | } |
| 304 | |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 305 | static inline int alloc_descs(unsigned int start, unsigned int cnt, int node, |
| 306 | struct module *owner) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 307 | { |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 308 | u32 i; |
| 309 | |
| 310 | for (i = 0; i < cnt; i++) { |
| 311 | struct irq_desc *desc = irq_to_desc(start + i); |
| 312 | |
| 313 | desc->owner = owner; |
| 314 | } |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 315 | return start; |
| 316 | } |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 317 | |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 318 | static int irq_expand_nr_irqs(unsigned int nr) |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 319 | { |
| 320 | return -ENOMEM; |
| 321 | } |
| 322 | |
Thomas Gleixner | f63b6a0 | 2014-05-07 15:44:21 +0000 | [diff] [blame] | 323 | void irq_mark_irq(unsigned int irq) |
| 324 | { |
| 325 | mutex_lock(&sparse_irq_lock); |
| 326 | bitmap_set(allocated_irqs, irq, 1); |
| 327 | mutex_unlock(&sparse_irq_lock); |
| 328 | } |
| 329 | |
Thomas Gleixner | c940e01 | 2014-05-07 15:44:22 +0000 | [diff] [blame] | 330 | #ifdef CONFIG_GENERIC_IRQ_LEGACY |
| 331 | void irq_init_desc(unsigned int irq) |
| 332 | { |
Thomas Gleixner | d8179bc | 2014-05-07 15:44:23 +0000 | [diff] [blame] | 333 | free_desc(irq); |
Thomas Gleixner | c940e01 | 2014-05-07 15:44:22 +0000 | [diff] [blame] | 334 | } |
| 335 | #endif |
| 336 | |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 337 | #endif /* !CONFIG_SPARSE_IRQ */ |
| 338 | |
Thomas Gleixner | fe12bc2 | 2011-05-18 12:48:00 +0200 | [diff] [blame] | 339 | /** |
| 340 | * generic_handle_irq - Invoke the handler for a particular irq |
| 341 | * @irq: The irq number to handle |
| 342 | * |
| 343 | */ |
| 344 | int generic_handle_irq(unsigned int irq) |
| 345 | { |
| 346 | struct irq_desc *desc = irq_to_desc(irq); |
| 347 | |
| 348 | if (!desc) |
| 349 | return -EINVAL; |
Thomas Gleixner | bd0b9ac | 2015-09-14 10:42:37 +0200 | [diff] [blame] | 350 | generic_handle_irq_desc(desc); |
Thomas Gleixner | fe12bc2 | 2011-05-18 12:48:00 +0200 | [diff] [blame] | 351 | return 0; |
| 352 | } |
Jonathan Cameron | edf76f8 | 2011-05-18 10:39:04 +0100 | [diff] [blame] | 353 | EXPORT_SYMBOL_GPL(generic_handle_irq); |
Thomas Gleixner | fe12bc2 | 2011-05-18 12:48:00 +0200 | [diff] [blame] | 354 | |
Marc Zyngier | 76ba59f | 2014-08-26 11:03:16 +0100 | [diff] [blame] | 355 | #ifdef CONFIG_HANDLE_DOMAIN_IRQ |
| 356 | /** |
| 357 | * __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain |
| 358 | * @domain: The domain where to perform the lookup |
| 359 | * @hwirq: The HW irq number to convert to a logical one |
| 360 | * @lookup: Whether to perform the domain lookup or not |
| 361 | * @regs: Register file coming from the low-level handling code |
| 362 | * |
| 363 | * Returns: 0 on success, or -EINVAL if conversion has failed |
| 364 | */ |
| 365 | int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq, |
| 366 | bool lookup, struct pt_regs *regs) |
| 367 | { |
| 368 | struct pt_regs *old_regs = set_irq_regs(regs); |
| 369 | unsigned int irq = hwirq; |
| 370 | int ret = 0; |
| 371 | |
| 372 | irq_enter(); |
| 373 | |
| 374 | #ifdef CONFIG_IRQ_DOMAIN |
| 375 | if (lookup) |
| 376 | irq = irq_find_mapping(domain, hwirq); |
| 377 | #endif |
| 378 | |
| 379 | /* |
| 380 | * Some hardware gives randomly wrong interrupts. Rather |
| 381 | * than crashing, do something sensible. |
| 382 | */ |
| 383 | if (unlikely(!irq || irq >= nr_irqs)) { |
| 384 | ack_bad_irq(irq); |
| 385 | ret = -EINVAL; |
| 386 | } else { |
| 387 | generic_handle_irq(irq); |
| 388 | } |
| 389 | |
| 390 | irq_exit(); |
| 391 | set_irq_regs(old_regs); |
| 392 | return ret; |
| 393 | } |
| 394 | #endif |
| 395 | |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 396 | /* Dynamic interrupt handling */ |
| 397 | |
| 398 | /** |
| 399 | * irq_free_descs - free irq descriptors |
| 400 | * @from: Start of descriptor range |
| 401 | * @cnt: Number of consecutive irqs to free |
| 402 | */ |
| 403 | void irq_free_descs(unsigned int from, unsigned int cnt) |
| 404 | { |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 405 | int i; |
| 406 | |
| 407 | if (from >= nr_irqs || (from + cnt) > nr_irqs) |
| 408 | return; |
| 409 | |
| 410 | for (i = 0; i < cnt; i++) |
| 411 | free_desc(from + i); |
| 412 | |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 413 | mutex_lock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 414 | bitmap_clear(allocated_irqs, from, cnt); |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 415 | mutex_unlock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 416 | } |
Jonathan Cameron | edf76f8 | 2011-05-18 10:39:04 +0100 | [diff] [blame] | 417 | EXPORT_SYMBOL_GPL(irq_free_descs); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 418 | |
| 419 | /** |
| 420 | * irq_alloc_descs - allocate and initialize a range of irq descriptors |
| 421 | * @irq: Allocate for specific irq number if irq >= 0 |
| 422 | * @from: Start the search from this irq number |
| 423 | * @cnt: Number of consecutive irqs to allocate. |
| 424 | * @node: Preferred node on which the irq descriptor should be allocated |
Randy Dunlap | d522a0d | 2011-08-18 12:19:27 -0700 | [diff] [blame] | 425 | * @owner: Owning module (can be NULL) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 426 | * |
| 427 | * Returns the first irq number or error code |
| 428 | */ |
| 429 | int __ref |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 430 | __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node, |
| 431 | struct module *owner) |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 432 | { |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 433 | int start, ret; |
| 434 | |
| 435 | if (!cnt) |
| 436 | return -EINVAL; |
| 437 | |
Mark Brown | c5182b8 | 2011-06-02 18:55:13 +0100 | [diff] [blame] | 438 | if (irq >= 0) { |
| 439 | if (from > irq) |
| 440 | return -EINVAL; |
| 441 | from = irq; |
Thomas Gleixner | 62a08ae | 2014-04-24 09:50:53 +0200 | [diff] [blame] | 442 | } else { |
| 443 | /* |
| 444 | * For interrupts which are freely allocated the |
| 445 | * architecture can force a lower bound to the @from |
| 446 | * argument. x86 uses this to exclude the GSI space. |
| 447 | */ |
| 448 | from = arch_dynirq_lower_bound(from); |
Mark Brown | c5182b8 | 2011-06-02 18:55:13 +0100 | [diff] [blame] | 449 | } |
| 450 | |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 451 | mutex_lock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 452 | |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 453 | start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS, |
| 454 | from, cnt, 0); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 455 | ret = -EEXIST; |
| 456 | if (irq >=0 && start != irq) |
| 457 | goto err; |
| 458 | |
Yinghai Lu | ed4dea6 | 2011-02-19 11:07:37 -0800 | [diff] [blame] | 459 | if (start + cnt > nr_irqs) { |
| 460 | ret = irq_expand_nr_irqs(start + cnt); |
Thomas Gleixner | e7bcecb | 2011-02-16 17:12:57 +0100 | [diff] [blame] | 461 | if (ret) |
| 462 | goto err; |
| 463 | } |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 464 | |
| 465 | bitmap_set(allocated_irqs, start, cnt); |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 466 | mutex_unlock(&sparse_irq_lock); |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 467 | return alloc_descs(start, cnt, node, owner); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 468 | |
| 469 | err: |
Thomas Gleixner | a05a900 | 2010-10-08 12:47:53 +0200 | [diff] [blame] | 470 | mutex_unlock(&sparse_irq_lock); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 471 | return ret; |
| 472 | } |
Sebastian Andrzej Siewior | b687380 | 2011-07-11 12:17:31 +0200 | [diff] [blame] | 473 | EXPORT_SYMBOL_GPL(__irq_alloc_descs); |
Thomas Gleixner | 1f5a5b8 | 2010-09-27 17:48:26 +0200 | [diff] [blame] | 474 | |
Thomas Gleixner | 7b6ef12 | 2014-05-07 15:44:05 +0000 | [diff] [blame] | 475 | #ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ |
| 476 | /** |
| 477 | * irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware |
| 478 | * @cnt: number of interrupts to allocate |
| 479 | * @node: node on which to allocate |
| 480 | * |
| 481 | * Returns an interrupt number > 0 or 0, if the allocation fails. |
| 482 | */ |
| 483 | unsigned int irq_alloc_hwirqs(int cnt, int node) |
| 484 | { |
| 485 | int i, irq = __irq_alloc_descs(-1, 0, cnt, node, NULL); |
| 486 | |
| 487 | if (irq < 0) |
| 488 | return 0; |
| 489 | |
| 490 | for (i = irq; cnt > 0; i++, cnt--) { |
| 491 | if (arch_setup_hwirq(i, node)) |
| 492 | goto err; |
| 493 | irq_clear_status_flags(i, _IRQ_NOREQUEST); |
| 494 | } |
| 495 | return irq; |
| 496 | |
| 497 | err: |
| 498 | for (i--; i >= irq; i--) { |
| 499 | irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE); |
| 500 | arch_teardown_hwirq(i); |
| 501 | } |
| 502 | irq_free_descs(irq, cnt); |
| 503 | return 0; |
| 504 | } |
| 505 | EXPORT_SYMBOL_GPL(irq_alloc_hwirqs); |
| 506 | |
| 507 | /** |
| 508 | * irq_free_hwirqs - Free irq descriptor and cleanup the hardware |
| 509 | * @from: Free from irq number |
| 510 | * @cnt: number of interrupts to free |
| 511 | * |
| 512 | */ |
| 513 | void irq_free_hwirqs(unsigned int from, int cnt) |
| 514 | { |
Keith Busch | 8844aad | 2014-06-30 16:24:44 -0600 | [diff] [blame] | 515 | int i, j; |
Thomas Gleixner | 7b6ef12 | 2014-05-07 15:44:05 +0000 | [diff] [blame] | 516 | |
Keith Busch | 8844aad | 2014-06-30 16:24:44 -0600 | [diff] [blame] | 517 | for (i = from, j = cnt; j > 0; i++, j--) { |
Thomas Gleixner | 7b6ef12 | 2014-05-07 15:44:05 +0000 | [diff] [blame] | 518 | irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE); |
| 519 | arch_teardown_hwirq(i); |
| 520 | } |
| 521 | irq_free_descs(from, cnt); |
| 522 | } |
| 523 | EXPORT_SYMBOL_GPL(irq_free_hwirqs); |
| 524 | #endif |
| 525 | |
Thomas Gleixner | a98d24b | 2010-09-30 10:45:07 +0200 | [diff] [blame] | 526 | /** |
| 527 | * irq_get_next_irq - get next allocated irq number |
| 528 | * @offset: where to start the search |
| 529 | * |
| 530 | * Returns next irq number after offset or nr_irqs if none is found. |
| 531 | */ |
| 532 | unsigned int irq_get_next_irq(unsigned int offset) |
| 533 | { |
| 534 | return find_next_bit(allocated_irqs, nr_irqs, offset); |
| 535 | } |
| 536 | |
Thomas Gleixner | d5eb4ad2 | 2011-02-12 12:16:16 +0100 | [diff] [blame] | 537 | struct irq_desc * |
Marc Zyngier | 31d9d9b | 2011-09-23 17:03:06 +0100 | [diff] [blame] | 538 | __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus, |
| 539 | unsigned int check) |
Thomas Gleixner | d5eb4ad2 | 2011-02-12 12:16:16 +0100 | [diff] [blame] | 540 | { |
| 541 | struct irq_desc *desc = irq_to_desc(irq); |
| 542 | |
| 543 | if (desc) { |
Marc Zyngier | 31d9d9b | 2011-09-23 17:03:06 +0100 | [diff] [blame] | 544 | if (check & _IRQ_DESC_CHECK) { |
| 545 | if ((check & _IRQ_DESC_PERCPU) && |
| 546 | !irq_settings_is_per_cpu_devid(desc)) |
| 547 | return NULL; |
| 548 | |
| 549 | if (!(check & _IRQ_DESC_PERCPU) && |
| 550 | irq_settings_is_per_cpu_devid(desc)) |
| 551 | return NULL; |
| 552 | } |
| 553 | |
Thomas Gleixner | d5eb4ad2 | 2011-02-12 12:16:16 +0100 | [diff] [blame] | 554 | if (bus) |
| 555 | chip_bus_lock(desc); |
| 556 | raw_spin_lock_irqsave(&desc->lock, *flags); |
| 557 | } |
| 558 | return desc; |
| 559 | } |
| 560 | |
| 561 | void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus) |
| 562 | { |
| 563 | raw_spin_unlock_irqrestore(&desc->lock, flags); |
| 564 | if (bus) |
| 565 | chip_bus_sync_unlock(desc); |
| 566 | } |
| 567 | |
Marc Zyngier | 31d9d9b | 2011-09-23 17:03:06 +0100 | [diff] [blame] | 568 | int irq_set_percpu_devid(unsigned int irq) |
| 569 | { |
| 570 | struct irq_desc *desc = irq_to_desc(irq); |
| 571 | |
| 572 | if (!desc) |
| 573 | return -EINVAL; |
| 574 | |
| 575 | if (desc->percpu_enabled) |
| 576 | return -EINVAL; |
| 577 | |
| 578 | desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL); |
| 579 | |
| 580 | if (!desc->percpu_enabled) |
| 581 | return -ENOMEM; |
| 582 | |
| 583 | irq_set_percpu_devid_flags(irq); |
| 584 | return 0; |
| 585 | } |
| 586 | |
Thomas Gleixner | 792d001 | 2014-02-23 21:40:14 +0000 | [diff] [blame] | 587 | void kstat_incr_irq_this_cpu(unsigned int irq) |
| 588 | { |
Jiang Liu | b51bf95 | 2015-06-04 12:13:25 +0800 | [diff] [blame] | 589 | kstat_incr_irqs_this_cpu(irq_to_desc(irq)); |
Thomas Gleixner | 792d001 | 2014-02-23 21:40:14 +0000 | [diff] [blame] | 590 | } |
| 591 | |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 592 | /** |
| 593 | * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu |
| 594 | * @irq: The interrupt number |
| 595 | * @cpu: The cpu number |
| 596 | * |
| 597 | * Returns the sum of interrupt counts on @cpu since boot for |
| 598 | * @irq. The caller must ensure that the interrupt is not removed |
| 599 | * concurrently. |
| 600 | */ |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 601 | unsigned int kstat_irqs_cpu(unsigned int irq, int cpu) |
| 602 | { |
| 603 | struct irq_desc *desc = irq_to_desc(irq); |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 604 | |
| 605 | return desc && desc->kstat_irqs ? |
| 606 | *per_cpu_ptr(desc->kstat_irqs, cpu) : 0; |
Thomas Gleixner | 3795de2 | 2010-09-22 17:09:43 +0200 | [diff] [blame] | 607 | } |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 608 | |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 609 | /** |
| 610 | * kstat_irqs - Get the statistics for an interrupt |
| 611 | * @irq: The interrupt number |
| 612 | * |
| 613 | * Returns the sum of interrupt counts on all cpus since boot for |
| 614 | * @irq. The caller must ensure that the interrupt is not removed |
| 615 | * concurrently. |
| 616 | */ |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 617 | unsigned int kstat_irqs(unsigned int irq) |
| 618 | { |
| 619 | struct irq_desc *desc = irq_to_desc(irq); |
| 620 | int cpu; |
Nicholas Mc Guire | 5e9662f | 2015-05-03 10:48:50 +0200 | [diff] [blame] | 621 | unsigned int sum = 0; |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 622 | |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 623 | if (!desc || !desc->kstat_irqs) |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 624 | return 0; |
| 625 | for_each_possible_cpu(cpu) |
Eric Dumazet | 6c9ae00 | 2011-01-13 15:45:38 -0800 | [diff] [blame] | 626 | sum += *per_cpu_ptr(desc->kstat_irqs, cpu); |
KAMEZAWA Hiroyuki | 478735e3 | 2010-10-27 15:34:15 -0700 | [diff] [blame] | 627 | return sum; |
| 628 | } |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 629 | |
| 630 | /** |
| 631 | * kstat_irqs_usr - Get the statistics for an interrupt |
| 632 | * @irq: The interrupt number |
| 633 | * |
| 634 | * Returns the sum of interrupt counts on all cpus since boot for |
| 635 | * @irq. Contrary to kstat_irqs() this can be called from any |
| 636 | * preemptible context. It's protected against concurrent removal of |
| 637 | * an interrupt descriptor when sparse irqs are enabled. |
| 638 | */ |
| 639 | unsigned int kstat_irqs_usr(unsigned int irq) |
| 640 | { |
Nicholas Mc Guire | 7df0b27 | 2015-05-03 10:49:11 +0200 | [diff] [blame] | 641 | unsigned int sum; |
Thomas Gleixner | c291ee6 | 2014-12-11 23:01:41 +0100 | [diff] [blame] | 642 | |
| 643 | irq_lock_sparse(); |
| 644 | sum = kstat_irqs(irq); |
| 645 | irq_unlock_sparse(); |
| 646 | return sum; |
| 647 | } |