1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Support for adapter interruptions 4 * 5 * Copyright IBM Corp. 1999, 2007 6 * Author(s): Ingo Adlung <adlung@de.ibm.com> 7 * Cornelia Huck <cornelia.huck@de.ibm.com> 8 * Arnd Bergmann <arndb@de.ibm.com> 9 * Peter Oberparleiter <peter.oberparleiter@de.ibm.com> 10 */ 11 12 #include <linux/init.h> 13 #include <linux/irq.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/rculist.h> 18 #include <linux/slab.h> 19 #include <linux/dmapool.h> 20 21 #include <asm/airq.h> 22 #include <asm/isc.h> 23 #include <asm/cio.h> 24 25 #include "cio.h" 26 #include "cio_debug.h" 27 #include "ioasm.h" 28 29 static DEFINE_SPINLOCK(airq_lists_lock); 30 static struct hlist_head airq_lists[MAX_ISC+1]; 31 32 static struct dma_pool *airq_iv_cache; 33 34 /** 35 * register_adapter_interrupt() - register adapter interrupt handler 36 * @airq: pointer to adapter interrupt descriptor 37 * 38 * Returns 0 on success, or -EINVAL. 39 */ 40 int register_adapter_interrupt(struct airq_struct *airq) 41 { 42 char dbf_txt[32]; 43 44 if (!airq->handler || airq->isc > MAX_ISC) 45 return -EINVAL; 46 if (!airq->lsi_ptr) { 47 airq->lsi_ptr = cio_dma_zalloc(1); 48 if (!airq->lsi_ptr) 49 return -ENOMEM; 50 airq->flags |= AIRQ_PTR_ALLOCATED; 51 } 52 if (!airq->lsi_mask) 53 airq->lsi_mask = 0xff; 54 snprintf(dbf_txt, sizeof(dbf_txt), "rairq:%p", airq); 55 CIO_TRACE_EVENT(4, dbf_txt); 56 isc_register(airq->isc); 57 spin_lock(&airq_lists_lock); 58 hlist_add_head_rcu(&airq->list, &airq_lists[airq->isc]); 59 spin_unlock(&airq_lists_lock); 60 return 0; 61 } 62 EXPORT_SYMBOL(register_adapter_interrupt); 63 64 /** 65 * unregister_adapter_interrupt - unregister adapter interrupt handler 66 * @airq: pointer to adapter interrupt descriptor 67 */ 68 void unregister_adapter_interrupt(struct airq_struct *airq) 69 { 70 char dbf_txt[32]; 71 72 if (hlist_unhashed(&airq->list)) 73 return; 74 snprintf(dbf_txt, sizeof(dbf_txt), "urairq:%p", airq); 75 CIO_TRACE_EVENT(4, dbf_txt); 76 spin_lock(&airq_lists_lock); 77 hlist_del_rcu(&airq->list); 78 spin_unlock(&airq_lists_lock); 79 synchronize_rcu(); 80 isc_unregister(airq->isc); 81 if (airq->flags & AIRQ_PTR_ALLOCATED) { 82 cio_dma_free(airq->lsi_ptr, 1); 83 airq->lsi_ptr = NULL; 84 airq->flags &= ~AIRQ_PTR_ALLOCATED; 85 } 86 } 87 EXPORT_SYMBOL(unregister_adapter_interrupt); 88 89 static irqreturn_t do_airq_interrupt(int irq, void *dummy) 90 { 91 struct tpi_info *tpi_info; 92 struct airq_struct *airq; 93 struct hlist_head *head; 94 95 set_cpu_flag(CIF_NOHZ_DELAY); 96 tpi_info = &get_irq_regs()->tpi_info; 97 trace_s390_cio_adapter_int(tpi_info); 98 head = &airq_lists[tpi_info->isc]; 99 rcu_read_lock(); 100 hlist_for_each_entry_rcu(airq, head, list) 101 if ((*airq->lsi_ptr & airq->lsi_mask) != 0) 102 airq->handler(airq, tpi_info); 103 rcu_read_unlock(); 104 105 return IRQ_HANDLED; 106 } 107 108 void __init init_airq_interrupts(void) 109 { 110 irq_set_chip_and_handler(THIN_INTERRUPT, 111 &dummy_irq_chip, handle_percpu_irq); 112 if (request_irq(THIN_INTERRUPT, do_airq_interrupt, 0, "AIO", NULL)) 113 panic("Failed to register AIO interrupt\n"); 114 } 115 116 static inline unsigned long iv_size(unsigned long bits) 117 { 118 return BITS_TO_LONGS(bits) * sizeof(unsigned long); 119 } 120 121 /** 122 * airq_iv_create - create an interrupt vector 123 * @bits: number of bits in the interrupt vector 124 * @flags: allocation flags 125 * @vec: pointer to pinned guest memory if AIRQ_IV_GUESTVEC 126 * 127 * Returns a pointer to an interrupt vector structure 128 */ 129 struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags, 130 unsigned long *vec) 131 { 132 struct airq_iv *iv; 133 unsigned long size; 134 135 iv = kzalloc(sizeof(*iv), GFP_KERNEL); 136 if (!iv) 137 goto out; 138 iv->bits = bits; 139 iv->flags = flags; 140 size = iv_size(bits); 141 142 if (flags & AIRQ_IV_CACHELINE) { 143 if ((cache_line_size() * BITS_PER_BYTE) < bits 144 || !airq_iv_cache) 145 goto out_free; 146 147 iv->vector = dma_pool_zalloc(airq_iv_cache, GFP_KERNEL, 148 &iv->vector_dma); 149 if (!iv->vector) 150 goto out_free; 151 } else if (flags & AIRQ_IV_GUESTVEC) { 152 iv->vector = vec; 153 } else { 154 iv->vector = cio_dma_zalloc(size); 155 if (!iv->vector) 156 goto out_free; 157 } 158 if (flags & AIRQ_IV_ALLOC) { 159 iv->avail = kmalloc(size, GFP_KERNEL); 160 if (!iv->avail) 161 goto out_free; 162 memset(iv->avail, 0xff, size); 163 iv->end = 0; 164 } else 165 iv->end = bits; 166 if (flags & AIRQ_IV_BITLOCK) { 167 iv->bitlock = kzalloc(size, GFP_KERNEL); 168 if (!iv->bitlock) 169 goto out_free; 170 } 171 if (flags & AIRQ_IV_PTR) { 172 size = bits * sizeof(unsigned long); 173 iv->ptr = kzalloc(size, GFP_KERNEL); 174 if (!iv->ptr) 175 goto out_free; 176 } 177 if (flags & AIRQ_IV_DATA) { 178 size = bits * sizeof(unsigned int); 179 iv->data = kzalloc(size, GFP_KERNEL); 180 if (!iv->data) 181 goto out_free; 182 } 183 spin_lock_init(&iv->lock); 184 return iv; 185 186 out_free: 187 kfree(iv->ptr); 188 kfree(iv->bitlock); 189 kfree(iv->avail); 190 if (iv->flags & AIRQ_IV_CACHELINE && iv->vector) 191 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma); 192 else if (!(iv->flags & AIRQ_IV_GUESTVEC)) 193 cio_dma_free(iv->vector, size); 194 kfree(iv); 195 out: 196 return NULL; 197 } 198 EXPORT_SYMBOL(airq_iv_create); 199 200 /** 201 * airq_iv_release - release an interrupt vector 202 * @iv: pointer to interrupt vector structure 203 */ 204 void airq_iv_release(struct airq_iv *iv) 205 { 206 kfree(iv->data); 207 kfree(iv->ptr); 208 kfree(iv->bitlock); 209 if (iv->flags & AIRQ_IV_CACHELINE) 210 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma); 211 else if (!(iv->flags & AIRQ_IV_GUESTVEC)) 212 cio_dma_free(iv->vector, iv_size(iv->bits)); 213 kfree(iv->avail); 214 kfree(iv); 215 } 216 EXPORT_SYMBOL(airq_iv_release); 217 218 /** 219 * airq_iv_alloc - allocate irq bits from an interrupt vector 220 * @iv: pointer to an interrupt vector structure 221 * @num: number of consecutive irq bits to allocate 222 * 223 * Returns the bit number of the first irq in the allocated block of irqs, 224 * or -1UL if no bit is available or the AIRQ_IV_ALLOC flag has not been 225 * specified 226 */ 227 unsigned long airq_iv_alloc(struct airq_iv *iv, unsigned long num) 228 { 229 unsigned long bit, i, flags; 230 231 if (!iv->avail || num == 0) 232 return -1UL; 233 spin_lock_irqsave(&iv->lock, flags); 234 bit = find_first_bit_inv(iv->avail, iv->bits); 235 while (bit + num <= iv->bits) { 236 for (i = 1; i < num; i++) 237 if (!test_bit_inv(bit + i, iv->avail)) 238 break; 239 if (i >= num) { 240 /* Found a suitable block of irqs */ 241 for (i = 0; i < num; i++) 242 clear_bit_inv(bit + i, iv->avail); 243 if (bit + num >= iv->end) 244 iv->end = bit + num + 1; 245 break; 246 } 247 bit = find_next_bit_inv(iv->avail, iv->bits, bit + i + 1); 248 } 249 if (bit + num > iv->bits) 250 bit = -1UL; 251 spin_unlock_irqrestore(&iv->lock, flags); 252 return bit; 253 } 254 EXPORT_SYMBOL(airq_iv_alloc); 255 256 /** 257 * airq_iv_free - free irq bits of an interrupt vector 258 * @iv: pointer to interrupt vector structure 259 * @bit: number of the first irq bit to free 260 * @num: number of consecutive irq bits to free 261 */ 262 void airq_iv_free(struct airq_iv *iv, unsigned long bit, unsigned long num) 263 { 264 unsigned long i, flags; 265 266 if (!iv->avail || num == 0) 267 return; 268 spin_lock_irqsave(&iv->lock, flags); 269 for (i = 0; i < num; i++) { 270 /* Clear (possibly left over) interrupt bit */ 271 clear_bit_inv(bit + i, iv->vector); 272 /* Make the bit positions available again */ 273 set_bit_inv(bit + i, iv->avail); 274 } 275 if (bit + num >= iv->end) { 276 /* Find new end of bit-field */ 277 while (iv->end > 0 && !test_bit_inv(iv->end - 1, iv->avail)) 278 iv->end--; 279 } 280 spin_unlock_irqrestore(&iv->lock, flags); 281 } 282 EXPORT_SYMBOL(airq_iv_free); 283 284 /** 285 * airq_iv_scan - scan interrupt vector for non-zero bits 286 * @iv: pointer to interrupt vector structure 287 * @start: bit number to start the search 288 * @end: bit number to end the search 289 * 290 * Returns the bit number of the next non-zero interrupt bit, or 291 * -1UL if the scan completed without finding any more any non-zero bits. 292 */ 293 unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start, 294 unsigned long end) 295 { 296 unsigned long bit; 297 298 /* Find non-zero bit starting from 'ivs->next'. */ 299 bit = find_next_bit_inv(iv->vector, end, start); 300 if (bit >= end) 301 return -1UL; 302 clear_bit_inv(bit, iv->vector); 303 return bit; 304 } 305 EXPORT_SYMBOL(airq_iv_scan); 306 307 int __init airq_init(void) 308 { 309 airq_iv_cache = dma_pool_create("airq_iv_cache", cio_get_dma_css_dev(), 310 cache_line_size(), 311 cache_line_size(), PAGE_SIZE); 312 if (!airq_iv_cache) 313 return -ENOMEM; 314 return 0; 315 } 316