1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2000, 2009 4 * Author(s): Utz Bacher <utz.bacher@de.ibm.com> 5 * Cornelia Huck <cornelia.huck@de.ibm.com> 6 * Jan Glauber <jang@linux.vnet.ibm.com> 7 */ 8 #include <linux/io.h> 9 #include <linux/slab.h> 10 #include <linux/kernel_stat.h> 11 #include <linux/atomic.h> 12 #include <linux/rculist.h> 13 14 #include <asm/debug.h> 15 #include <asm/qdio.h> 16 #include <asm/airq.h> 17 #include <asm/isc.h> 18 19 #include "cio.h" 20 #include "ioasm.h" 21 #include "qdio.h" 22 #include "qdio_debug.h" 23 24 /* 25 * Restriction: only 63 iqdio subchannels would have its own indicator, 26 * after that, subsequent subchannels share one indicator 27 */ 28 #define TIQDIO_NR_NONSHARED_IND 63 29 #define TIQDIO_NR_INDICATORS (TIQDIO_NR_NONSHARED_IND + 1) 30 #define TIQDIO_SHARED_IND 63 31 32 /* device state change indicators */ 33 struct indicator_t { 34 u32 ind; /* u32 because of compare-and-swap performance */ 35 atomic_t count; /* use count, 0 or 1 for non-shared indicators */ 36 }; 37 38 /* list of thin interrupt input queues */ 39 static LIST_HEAD(tiq_list); 40 static DEFINE_MUTEX(tiq_list_lock); 41 42 static struct indicator_t *q_indicators; 43 44 u64 last_ai_time; 45 46 /* returns addr for the device state change indicator */ 47 static u32 *get_indicator(void) 48 { 49 int i; 50 51 for (i = 0; i < TIQDIO_NR_NONSHARED_IND; i++) 52 if (!atomic_cmpxchg(&q_indicators[i].count, 0, 1)) 53 return &q_indicators[i].ind; 54 55 /* use the shared indicator */ 56 atomic_inc(&q_indicators[TIQDIO_SHARED_IND].count); 57 return &q_indicators[TIQDIO_SHARED_IND].ind; 58 } 59 60 static void put_indicator(u32 *addr) 61 { 62 struct indicator_t *ind = container_of(addr, struct indicator_t, ind); 63 64 if (!addr) 65 return; 66 atomic_dec(&ind->count); 67 } 68 69 void tiqdio_add_device(struct qdio_irq *irq_ptr) 70 { 71 mutex_lock(&tiq_list_lock); 72 list_add_rcu(&irq_ptr->entry, &tiq_list); 73 mutex_unlock(&tiq_list_lock); 74 } 75 76 void tiqdio_remove_device(struct qdio_irq *irq_ptr) 77 { 78 mutex_lock(&tiq_list_lock); 79 list_del_rcu(&irq_ptr->entry); 80 mutex_unlock(&tiq_list_lock); 81 synchronize_rcu(); 82 INIT_LIST_HEAD(&irq_ptr->entry); 83 } 84 85 static inline int has_multiple_inq_on_dsci(struct qdio_irq *irq_ptr) 86 { 87 return irq_ptr->nr_input_qs > 1; 88 } 89 90 static inline int references_shared_dsci(struct qdio_irq *irq_ptr) 91 { 92 return irq_ptr->dsci == &q_indicators[TIQDIO_SHARED_IND].ind; 93 } 94 95 static inline int shared_ind(struct qdio_irq *irq_ptr) 96 { 97 return references_shared_dsci(irq_ptr) || 98 has_multiple_inq_on_dsci(irq_ptr); 99 } 100 101 void clear_nonshared_ind(struct qdio_irq *irq_ptr) 102 { 103 if (!is_thinint_irq(irq_ptr)) 104 return; 105 if (shared_ind(irq_ptr)) 106 return; 107 xchg(irq_ptr->dsci, 0); 108 } 109 110 int test_nonshared_ind(struct qdio_irq *irq_ptr) 111 { 112 if (!is_thinint_irq(irq_ptr)) 113 return 0; 114 if (shared_ind(irq_ptr)) 115 return 0; 116 if (*irq_ptr->dsci) 117 return 1; 118 else 119 return 0; 120 } 121 122 static inline u32 clear_shared_ind(void) 123 { 124 if (!atomic_read(&q_indicators[TIQDIO_SHARED_IND].count)) 125 return 0; 126 return xchg(&q_indicators[TIQDIO_SHARED_IND].ind, 0); 127 } 128 129 static inline void tiqdio_call_inq_handlers(struct qdio_irq *irq) 130 { 131 struct qdio_q *q; 132 int i; 133 134 if (!references_shared_dsci(irq) && 135 has_multiple_inq_on_dsci(irq)) 136 xchg(irq->dsci, 0); 137 138 for_each_input_queue(irq, q, i) { 139 if (q->u.in.queue_start_poll) { 140 /* skip if polling is enabled or already in work */ 141 if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED, 142 &q->u.in.queue_irq_state)) { 143 QDIO_PERF_STAT_INC(irq, int_discarded); 144 continue; 145 } 146 147 /* avoid dsci clear here, done after processing */ 148 q->u.in.queue_start_poll(irq->cdev, q->nr, 149 irq->int_parm); 150 } else { 151 if (!shared_ind(irq)) 152 xchg(irq->dsci, 0); 153 154 /* 155 * Call inbound processing but not directly 156 * since that could starve other thinint queues. 157 */ 158 tasklet_schedule(&q->tasklet); 159 } 160 } 161 } 162 163 /** 164 * tiqdio_thinint_handler - thin interrupt handler for qdio 165 * @airq: pointer to adapter interrupt descriptor 166 * @floating: flag to recognize floating vs. directed interrupts (unused) 167 */ 168 static void tiqdio_thinint_handler(struct airq_struct *airq, bool floating) 169 { 170 u32 si_used = clear_shared_ind(); 171 struct qdio_irq *irq; 172 173 last_ai_time = S390_lowcore.int_clock; 174 inc_irq_stat(IRQIO_QAI); 175 176 /* protect tiq_list entries, only changed in activate or shutdown */ 177 rcu_read_lock(); 178 179 list_for_each_entry_rcu(irq, &tiq_list, entry) { 180 /* only process queues from changed sets */ 181 if (unlikely(references_shared_dsci(irq))) { 182 if (!si_used) 183 continue; 184 } else if (!*irq->dsci) 185 continue; 186 187 tiqdio_call_inq_handlers(irq); 188 189 QDIO_PERF_STAT_INC(irq, adapter_int); 190 } 191 rcu_read_unlock(); 192 } 193 194 static struct airq_struct tiqdio_airq = { 195 .handler = tiqdio_thinint_handler, 196 .isc = QDIO_AIRQ_ISC, 197 }; 198 199 static int set_subchannel_ind(struct qdio_irq *irq_ptr, int reset) 200 { 201 struct chsc_scssc_area *scssc = (void *)irq_ptr->chsc_page; 202 u64 summary_indicator_addr, subchannel_indicator_addr; 203 int rc; 204 205 if (reset) { 206 summary_indicator_addr = 0; 207 subchannel_indicator_addr = 0; 208 } else { 209 summary_indicator_addr = virt_to_phys(tiqdio_airq.lsi_ptr); 210 subchannel_indicator_addr = virt_to_phys(irq_ptr->dsci); 211 } 212 213 rc = chsc_sadc(irq_ptr->schid, scssc, summary_indicator_addr, 214 subchannel_indicator_addr); 215 if (rc) { 216 DBF_ERROR("%4x SSI r:%4x", irq_ptr->schid.sch_no, 217 scssc->response.code); 218 goto out; 219 } 220 221 DBF_EVENT("setscind"); 222 DBF_HEX(&summary_indicator_addr, sizeof(summary_indicator_addr)); 223 DBF_HEX(&subchannel_indicator_addr, sizeof(subchannel_indicator_addr)); 224 out: 225 return rc; 226 } 227 228 /* allocate non-shared indicators and shared indicator */ 229 int __init tiqdio_allocate_memory(void) 230 { 231 q_indicators = kcalloc(TIQDIO_NR_INDICATORS, 232 sizeof(struct indicator_t), 233 GFP_KERNEL); 234 if (!q_indicators) 235 return -ENOMEM; 236 return 0; 237 } 238 239 void tiqdio_free_memory(void) 240 { 241 kfree(q_indicators); 242 } 243 244 int __init tiqdio_register_thinints(void) 245 { 246 int rc; 247 248 rc = register_adapter_interrupt(&tiqdio_airq); 249 if (rc) { 250 DBF_EVENT("RTI:%x", rc); 251 return rc; 252 } 253 return 0; 254 } 255 256 int qdio_establish_thinint(struct qdio_irq *irq_ptr) 257 { 258 if (!is_thinint_irq(irq_ptr)) 259 return 0; 260 return set_subchannel_ind(irq_ptr, 0); 261 } 262 263 void qdio_setup_thinint(struct qdio_irq *irq_ptr) 264 { 265 if (!is_thinint_irq(irq_ptr)) 266 return; 267 irq_ptr->dsci = get_indicator(); 268 DBF_HEX(&irq_ptr->dsci, sizeof(void *)); 269 } 270 271 void qdio_shutdown_thinint(struct qdio_irq *irq_ptr) 272 { 273 if (!is_thinint_irq(irq_ptr)) 274 return; 275 276 /* reset adapter interrupt indicators */ 277 set_subchannel_ind(irq_ptr, 1); 278 put_indicator(irq_ptr->dsci); 279 } 280 281 void __exit tiqdio_unregister_thinints(void) 282 { 283 WARN_ON(!list_empty(&tiq_list)); 284 unregister_adapter_interrupt(&tiqdio_airq); 285 } 286