1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2022, Linaro Limited, All rights reserved. 4 * Author: Mike Leach <mike.leach@linaro.org> 5 */ 6 #include <linux/coresight-pmu.h> 7 #include <linux/cpumask.h> 8 #include <linux/kernel.h> 9 #include <linux/spinlock.h> 10 #include <linux/types.h> 11 12 #include "coresight-trace-id.h" 13 14 /* Default trace ID map. Used on systems that don't require per sink mappings */ 15 static struct coresight_trace_id_map id_map_default; 16 17 /* maintain a record of the mapping of IDs and pending releases per cpu */ 18 static DEFINE_PER_CPU(atomic_t, cpu_id) = ATOMIC_INIT(0); 19 static cpumask_t cpu_id_release_pending; 20 21 /* perf session active counter */ 22 static atomic_t perf_cs_etm_session_active = ATOMIC_INIT(0); 23 24 /* lock to protect id_map and cpu data */ 25 static DEFINE_SPINLOCK(id_map_lock); 26 27 /* #define TRACE_ID_DEBUG 1 */ 28 #if defined(TRACE_ID_DEBUG) || defined(CONFIG_COMPILE_TEST) 29 30 static void coresight_trace_id_dump_table(struct coresight_trace_id_map *id_map, 31 const char *func_name) 32 { 33 pr_debug("%s id_map::\n", func_name); 34 pr_debug("Used = %*pb\n", CORESIGHT_TRACE_IDS_MAX, id_map->used_ids); 35 pr_debug("Pend = %*pb\n", CORESIGHT_TRACE_IDS_MAX, id_map->pend_rel_ids); 36 } 37 #define DUMP_ID_MAP(map) coresight_trace_id_dump_table(map, __func__) 38 #define DUMP_ID_CPU(cpu, id) pr_debug("%s called; cpu=%d, id=%d\n", __func__, cpu, id) 39 #define DUMP_ID(id) pr_debug("%s called; id=%d\n", __func__, id) 40 #define PERF_SESSION(n) pr_debug("%s perf count %d\n", __func__, n) 41 #else 42 #define DUMP_ID_MAP(map) 43 #define DUMP_ID(id) 44 #define DUMP_ID_CPU(cpu, id) 45 #define PERF_SESSION(n) 46 #endif 47 48 /* unlocked read of current trace ID value for given CPU */ 49 static int _coresight_trace_id_read_cpu_id(int cpu) 50 { 51 return atomic_read(&per_cpu(cpu_id, cpu)); 52 } 53 54 /* look for next available odd ID, return 0 if none found */ 55 static int coresight_trace_id_find_odd_id(struct coresight_trace_id_map *id_map) 56 { 57 int found_id = 0, bit = 1, next_id; 58 59 while ((bit < CORESIGHT_TRACE_ID_RES_TOP) && !found_id) { 60 /* 61 * bitmap length of CORESIGHT_TRACE_ID_RES_TOP, 62 * search from offset `bit`. 63 */ 64 next_id = find_next_zero_bit(id_map->used_ids, 65 CORESIGHT_TRACE_ID_RES_TOP, bit); 66 if ((next_id < CORESIGHT_TRACE_ID_RES_TOP) && (next_id & 0x1)) 67 found_id = next_id; 68 else 69 bit = next_id + 1; 70 } 71 return found_id; 72 } 73 74 /* 75 * Allocate new ID and set in use 76 * 77 * if @preferred_id is a valid id then try to use that value if available. 78 * if @preferred_id is not valid and @prefer_odd_id is true, try for odd id. 79 * 80 * Otherwise allocate next available ID. 81 */ 82 static int coresight_trace_id_alloc_new_id(struct coresight_trace_id_map *id_map, 83 int preferred_id, bool prefer_odd_id) 84 { 85 int id = 0; 86 87 /* for backwards compatibility, cpu IDs may use preferred value */ 88 if (IS_VALID_CS_TRACE_ID(preferred_id) && 89 !test_bit(preferred_id, id_map->used_ids)) { 90 id = preferred_id; 91 goto trace_id_allocated; 92 } else if (prefer_odd_id) { 93 /* may use odd ids to avoid preferred legacy cpu IDs */ 94 id = coresight_trace_id_find_odd_id(id_map); 95 if (id) 96 goto trace_id_allocated; 97 } 98 99 /* 100 * skip reserved bit 0, look at bitmap length of 101 * CORESIGHT_TRACE_ID_RES_TOP from offset of bit 1. 102 */ 103 id = find_next_zero_bit(id_map->used_ids, CORESIGHT_TRACE_ID_RES_TOP, 1); 104 if (id >= CORESIGHT_TRACE_ID_RES_TOP) 105 return -EINVAL; 106 107 /* mark as used */ 108 trace_id_allocated: 109 set_bit(id, id_map->used_ids); 110 return id; 111 } 112 113 static void coresight_trace_id_free(int id, struct coresight_trace_id_map *id_map) 114 { 115 if (WARN(!IS_VALID_CS_TRACE_ID(id), "Invalid Trace ID %d\n", id)) 116 return; 117 if (WARN(!test_bit(id, id_map->used_ids), "Freeing unused ID %d\n", id)) 118 return; 119 clear_bit(id, id_map->used_ids); 120 } 121 122 static void coresight_trace_id_set_pend_rel(int id, struct coresight_trace_id_map *id_map) 123 { 124 if (WARN(!IS_VALID_CS_TRACE_ID(id), "Invalid Trace ID %d\n", id)) 125 return; 126 set_bit(id, id_map->pend_rel_ids); 127 } 128 129 /* 130 * release all pending IDs for all current maps & clear CPU associations 131 * 132 * This currently operates on the default id map, but may be extended to 133 * operate on all registered id maps if per sink id maps are used. 134 */ 135 static void coresight_trace_id_release_all_pending(void) 136 { 137 struct coresight_trace_id_map *id_map = &id_map_default; 138 unsigned long flags; 139 int cpu, bit; 140 141 spin_lock_irqsave(&id_map_lock, flags); 142 for_each_set_bit(bit, id_map->pend_rel_ids, CORESIGHT_TRACE_ID_RES_TOP) { 143 clear_bit(bit, id_map->used_ids); 144 clear_bit(bit, id_map->pend_rel_ids); 145 } 146 for_each_cpu(cpu, &cpu_id_release_pending) { 147 atomic_set(&per_cpu(cpu_id, cpu), 0); 148 cpumask_clear_cpu(cpu, &cpu_id_release_pending); 149 } 150 spin_unlock_irqrestore(&id_map_lock, flags); 151 DUMP_ID_MAP(id_map); 152 } 153 154 static int coresight_trace_id_map_get_cpu_id(int cpu, struct coresight_trace_id_map *id_map) 155 { 156 unsigned long flags; 157 int id; 158 159 spin_lock_irqsave(&id_map_lock, flags); 160 161 /* check for existing allocation for this CPU */ 162 id = _coresight_trace_id_read_cpu_id(cpu); 163 if (id) 164 goto get_cpu_id_clr_pend; 165 166 /* 167 * Find a new ID. 168 * 169 * Use legacy values where possible in the dynamic trace ID allocator to 170 * allow older tools to continue working if they are not upgraded at the 171 * same time as the kernel drivers. 172 * 173 * If the generated legacy ID is invalid, or not available then the next 174 * available dynamic ID will be used. 175 */ 176 id = coresight_trace_id_alloc_new_id(id_map, 177 CORESIGHT_LEGACY_CPU_TRACE_ID(cpu), 178 false); 179 if (!IS_VALID_CS_TRACE_ID(id)) 180 goto get_cpu_id_out_unlock; 181 182 /* allocate the new id to the cpu */ 183 atomic_set(&per_cpu(cpu_id, cpu), id); 184 185 get_cpu_id_clr_pend: 186 /* we are (re)using this ID - so ensure it is not marked for release */ 187 cpumask_clear_cpu(cpu, &cpu_id_release_pending); 188 clear_bit(id, id_map->pend_rel_ids); 189 190 get_cpu_id_out_unlock: 191 spin_unlock_irqrestore(&id_map_lock, flags); 192 193 DUMP_ID_CPU(cpu, id); 194 DUMP_ID_MAP(id_map); 195 return id; 196 } 197 198 static void coresight_trace_id_map_put_cpu_id(int cpu, struct coresight_trace_id_map *id_map) 199 { 200 unsigned long flags; 201 int id; 202 203 /* check for existing allocation for this CPU */ 204 id = _coresight_trace_id_read_cpu_id(cpu); 205 if (!id) 206 return; 207 208 spin_lock_irqsave(&id_map_lock, flags); 209 210 if (atomic_read(&perf_cs_etm_session_active)) { 211 /* set release at pending if perf still active */ 212 coresight_trace_id_set_pend_rel(id, id_map); 213 cpumask_set_cpu(cpu, &cpu_id_release_pending); 214 } else { 215 /* otherwise clear id */ 216 coresight_trace_id_free(id, id_map); 217 atomic_set(&per_cpu(cpu_id, cpu), 0); 218 } 219 220 spin_unlock_irqrestore(&id_map_lock, flags); 221 DUMP_ID_CPU(cpu, id); 222 DUMP_ID_MAP(id_map); 223 } 224 225 static int coresight_trace_id_map_get_system_id(struct coresight_trace_id_map *id_map) 226 { 227 unsigned long flags; 228 int id; 229 230 spin_lock_irqsave(&id_map_lock, flags); 231 /* prefer odd IDs for system components to avoid legacy CPU IDS */ 232 id = coresight_trace_id_alloc_new_id(id_map, 0, true); 233 spin_unlock_irqrestore(&id_map_lock, flags); 234 235 DUMP_ID(id); 236 DUMP_ID_MAP(id_map); 237 return id; 238 } 239 240 static void coresight_trace_id_map_put_system_id(struct coresight_trace_id_map *id_map, int id) 241 { 242 unsigned long flags; 243 244 spin_lock_irqsave(&id_map_lock, flags); 245 coresight_trace_id_free(id, id_map); 246 spin_unlock_irqrestore(&id_map_lock, flags); 247 248 DUMP_ID(id); 249 DUMP_ID_MAP(id_map); 250 } 251 252 /* API functions */ 253 254 int coresight_trace_id_get_cpu_id(int cpu) 255 { 256 return coresight_trace_id_map_get_cpu_id(cpu, &id_map_default); 257 } 258 EXPORT_SYMBOL_GPL(coresight_trace_id_get_cpu_id); 259 260 void coresight_trace_id_put_cpu_id(int cpu) 261 { 262 coresight_trace_id_map_put_cpu_id(cpu, &id_map_default); 263 } 264 EXPORT_SYMBOL_GPL(coresight_trace_id_put_cpu_id); 265 266 int coresight_trace_id_read_cpu_id(int cpu) 267 { 268 return _coresight_trace_id_read_cpu_id(cpu); 269 } 270 EXPORT_SYMBOL_GPL(coresight_trace_id_read_cpu_id); 271 272 int coresight_trace_id_get_system_id(void) 273 { 274 return coresight_trace_id_map_get_system_id(&id_map_default); 275 } 276 EXPORT_SYMBOL_GPL(coresight_trace_id_get_system_id); 277 278 void coresight_trace_id_put_system_id(int id) 279 { 280 coresight_trace_id_map_put_system_id(&id_map_default, id); 281 } 282 EXPORT_SYMBOL_GPL(coresight_trace_id_put_system_id); 283 284 void coresight_trace_id_perf_start(void) 285 { 286 atomic_inc(&perf_cs_etm_session_active); 287 PERF_SESSION(atomic_read(&perf_cs_etm_session_active)); 288 } 289 EXPORT_SYMBOL_GPL(coresight_trace_id_perf_start); 290 291 void coresight_trace_id_perf_stop(void) 292 { 293 if (!atomic_dec_return(&perf_cs_etm_session_active)) 294 coresight_trace_id_release_all_pending(); 295 PERF_SESSION(atomic_read(&perf_cs_etm_session_active)); 296 } 297 EXPORT_SYMBOL_GPL(coresight_trace_id_perf_stop); 298