1 // SPDX-License-Identifier: GPL-2.0 2 #define pr_fmt(fmt) "kcov: " fmt 3 4 #define DISABLE_BRANCH_PROFILING 5 #include <linux/atomic.h> 6 #include <linux/compiler.h> 7 #include <linux/errno.h> 8 #include <linux/export.h> 9 #include <linux/types.h> 10 #include <linux/file.h> 11 #include <linux/fs.h> 12 #include <linux/init.h> 13 #include <linux/mm.h> 14 #include <linux/preempt.h> 15 #include <linux/printk.h> 16 #include <linux/sched.h> 17 #include <linux/slab.h> 18 #include <linux/spinlock.h> 19 #include <linux/vmalloc.h> 20 #include <linux/debugfs.h> 21 #include <linux/uaccess.h> 22 #include <linux/kcov.h> 23 #include <asm/setup.h> 24 25 /* Number of 64-bit words written per one comparison: */ 26 #define KCOV_WORDS_PER_CMP 4 27 28 /* 29 * kcov descriptor (one per opened debugfs file). 30 * State transitions of the descriptor: 31 * - initial state after open() 32 * - then there must be a single ioctl(KCOV_INIT_TRACE) call 33 * - then, mmap() call (several calls are allowed but not useful) 34 * - then, ioctl(KCOV_ENABLE, arg), where arg is 35 * KCOV_TRACE_PC - to trace only the PCs 36 * or 37 * KCOV_TRACE_CMP - to trace only the comparison operands 38 * - then, ioctl(KCOV_DISABLE) to disable the task. 39 * Enabling/disabling ioctls can be repeated (only one task a time allowed). 40 */ 41 struct kcov { 42 /* 43 * Reference counter. We keep one for: 44 * - opened file descriptor 45 * - task with enabled coverage (we can't unwire it from another task) 46 */ 47 atomic_t refcount; 48 /* The lock protects mode, size, area and t. */ 49 spinlock_t lock; 50 enum kcov_mode mode; 51 /* Size of arena (in long's for KCOV_MODE_TRACE). */ 52 unsigned size; 53 /* Coverage buffer shared with user space. */ 54 void *area; 55 /* Task for which we collect coverage, or NULL. */ 56 struct task_struct *t; 57 }; 58 59 static bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t) 60 { 61 enum kcov_mode mode; 62 63 /* 64 * We are interested in code coverage as a function of a syscall inputs, 65 * so we ignore code executed in interrupts. 66 */ 67 if (!in_task()) 68 return false; 69 mode = READ_ONCE(t->kcov_mode); 70 /* 71 * There is some code that runs in interrupts but for which 72 * in_interrupt() returns false (e.g. preempt_schedule_irq()). 73 * READ_ONCE()/barrier() effectively provides load-acquire wrt 74 * interrupts, there are paired barrier()/WRITE_ONCE() in 75 * kcov_ioctl_locked(). 76 */ 77 barrier(); 78 return mode == needed_mode; 79 } 80 81 static unsigned long canonicalize_ip(unsigned long ip) 82 { 83 #ifdef CONFIG_RANDOMIZE_BASE 84 ip -= kaslr_offset(); 85 #endif 86 return ip; 87 } 88 89 /* 90 * Entry point from instrumented code. 91 * This is called once per basic-block/edge. 92 */ 93 void notrace __sanitizer_cov_trace_pc(void) 94 { 95 struct task_struct *t; 96 unsigned long *area; 97 unsigned long ip = canonicalize_ip(_RET_IP_); 98 unsigned long pos; 99 100 t = current; 101 if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t)) 102 return; 103 104 area = t->kcov_area; 105 /* The first 64-bit word is the number of subsequent PCs. */ 106 pos = READ_ONCE(area[0]) + 1; 107 if (likely(pos < t->kcov_size)) { 108 area[pos] = ip; 109 WRITE_ONCE(area[0], pos); 110 } 111 } 112 EXPORT_SYMBOL(__sanitizer_cov_trace_pc); 113 114 #ifdef CONFIG_KCOV_ENABLE_COMPARISONS 115 static void write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip) 116 { 117 struct task_struct *t; 118 u64 *area; 119 u64 count, start_index, end_pos, max_pos; 120 121 t = current; 122 if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t)) 123 return; 124 125 ip = canonicalize_ip(ip); 126 127 /* 128 * We write all comparison arguments and types as u64. 129 * The buffer was allocated for t->kcov_size unsigned longs. 130 */ 131 area = (u64 *)t->kcov_area; 132 max_pos = t->kcov_size * sizeof(unsigned long); 133 134 count = READ_ONCE(area[0]); 135 136 /* Every record is KCOV_WORDS_PER_CMP 64-bit words. */ 137 start_index = 1 + count * KCOV_WORDS_PER_CMP; 138 end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64); 139 if (likely(end_pos <= max_pos)) { 140 area[start_index] = type; 141 area[start_index + 1] = arg1; 142 area[start_index + 2] = arg2; 143 area[start_index + 3] = ip; 144 WRITE_ONCE(area[0], count + 1); 145 } 146 } 147 148 void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2) 149 { 150 write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_); 151 } 152 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1); 153 154 void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2) 155 { 156 write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_); 157 } 158 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2); 159 160 void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2) 161 { 162 write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_); 163 } 164 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4); 165 166 void notrace __sanitizer_cov_trace_cmp8(u64 arg1, u64 arg2) 167 { 168 write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_); 169 } 170 EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8); 171 172 void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2) 173 { 174 write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2, 175 _RET_IP_); 176 } 177 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1); 178 179 void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2) 180 { 181 write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2, 182 _RET_IP_); 183 } 184 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2); 185 186 void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2) 187 { 188 write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2, 189 _RET_IP_); 190 } 191 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4); 192 193 void notrace __sanitizer_cov_trace_const_cmp8(u64 arg1, u64 arg2) 194 { 195 write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2, 196 _RET_IP_); 197 } 198 EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8); 199 200 void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases) 201 { 202 u64 i; 203 u64 count = cases[0]; 204 u64 size = cases[1]; 205 u64 type = KCOV_CMP_CONST; 206 207 switch (size) { 208 case 8: 209 type |= KCOV_CMP_SIZE(0); 210 break; 211 case 16: 212 type |= KCOV_CMP_SIZE(1); 213 break; 214 case 32: 215 type |= KCOV_CMP_SIZE(2); 216 break; 217 case 64: 218 type |= KCOV_CMP_SIZE(3); 219 break; 220 default: 221 return; 222 } 223 for (i = 0; i < count; i++) 224 write_comp_data(type, cases[i + 2], val, _RET_IP_); 225 } 226 EXPORT_SYMBOL(__sanitizer_cov_trace_switch); 227 #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */ 228 229 static void kcov_get(struct kcov *kcov) 230 { 231 atomic_inc(&kcov->refcount); 232 } 233 234 static void kcov_put(struct kcov *kcov) 235 { 236 if (atomic_dec_and_test(&kcov->refcount)) { 237 vfree(kcov->area); 238 kfree(kcov); 239 } 240 } 241 242 void kcov_task_init(struct task_struct *t) 243 { 244 t->kcov_mode = KCOV_MODE_DISABLED; 245 t->kcov_size = 0; 246 t->kcov_area = NULL; 247 t->kcov = NULL; 248 } 249 250 void kcov_task_exit(struct task_struct *t) 251 { 252 struct kcov *kcov; 253 254 kcov = t->kcov; 255 if (kcov == NULL) 256 return; 257 spin_lock(&kcov->lock); 258 if (WARN_ON(kcov->t != t)) { 259 spin_unlock(&kcov->lock); 260 return; 261 } 262 /* Just to not leave dangling references behind. */ 263 kcov_task_init(t); 264 kcov->t = NULL; 265 kcov->mode = KCOV_MODE_INIT; 266 spin_unlock(&kcov->lock); 267 kcov_put(kcov); 268 } 269 270 static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) 271 { 272 int res = 0; 273 void *area; 274 struct kcov *kcov = vma->vm_file->private_data; 275 unsigned long size, off; 276 struct page *page; 277 278 area = vmalloc_user(vma->vm_end - vma->vm_start); 279 if (!area) 280 return -ENOMEM; 281 282 spin_lock(&kcov->lock); 283 size = kcov->size * sizeof(unsigned long); 284 if (kcov->mode != KCOV_MODE_INIT || vma->vm_pgoff != 0 || 285 vma->vm_end - vma->vm_start != size) { 286 res = -EINVAL; 287 goto exit; 288 } 289 if (!kcov->area) { 290 kcov->area = area; 291 vma->vm_flags |= VM_DONTEXPAND; 292 spin_unlock(&kcov->lock); 293 for (off = 0; off < size; off += PAGE_SIZE) { 294 page = vmalloc_to_page(kcov->area + off); 295 if (vm_insert_page(vma, vma->vm_start + off, page)) 296 WARN_ONCE(1, "vm_insert_page() failed"); 297 } 298 return 0; 299 } 300 exit: 301 spin_unlock(&kcov->lock); 302 vfree(area); 303 return res; 304 } 305 306 static int kcov_open(struct inode *inode, struct file *filep) 307 { 308 struct kcov *kcov; 309 310 kcov = kzalloc(sizeof(*kcov), GFP_KERNEL); 311 if (!kcov) 312 return -ENOMEM; 313 kcov->mode = KCOV_MODE_DISABLED; 314 atomic_set(&kcov->refcount, 1); 315 spin_lock_init(&kcov->lock); 316 filep->private_data = kcov; 317 return nonseekable_open(inode, filep); 318 } 319 320 static int kcov_close(struct inode *inode, struct file *filep) 321 { 322 kcov_put(filep->private_data); 323 return 0; 324 } 325 326 static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, 327 unsigned long arg) 328 { 329 struct task_struct *t; 330 unsigned long size, unused; 331 332 switch (cmd) { 333 case KCOV_INIT_TRACE: 334 /* 335 * Enable kcov in trace mode and setup buffer size. 336 * Must happen before anything else. 337 */ 338 if (kcov->mode != KCOV_MODE_DISABLED) 339 return -EBUSY; 340 /* 341 * Size must be at least 2 to hold current position and one PC. 342 * Later we allocate size * sizeof(unsigned long) memory, 343 * that must not overflow. 344 */ 345 size = arg; 346 if (size < 2 || size > INT_MAX / sizeof(unsigned long)) 347 return -EINVAL; 348 kcov->size = size; 349 kcov->mode = KCOV_MODE_INIT; 350 return 0; 351 case KCOV_ENABLE: 352 /* 353 * Enable coverage for the current task. 354 * At this point user must have been enabled trace mode, 355 * and mmapped the file. Coverage collection is disabled only 356 * at task exit or voluntary by KCOV_DISABLE. After that it can 357 * be enabled for another task. 358 */ 359 if (kcov->mode != KCOV_MODE_INIT || !kcov->area) 360 return -EINVAL; 361 t = current; 362 if (kcov->t != NULL || t->kcov != NULL) 363 return -EBUSY; 364 if (arg == KCOV_TRACE_PC) 365 kcov->mode = KCOV_MODE_TRACE_PC; 366 else if (arg == KCOV_TRACE_CMP) 367 #ifdef CONFIG_KCOV_ENABLE_COMPARISONS 368 kcov->mode = KCOV_MODE_TRACE_CMP; 369 #else 370 return -ENOTSUPP; 371 #endif 372 else 373 return -EINVAL; 374 /* Cache in task struct for performance. */ 375 t->kcov_size = kcov->size; 376 t->kcov_area = kcov->area; 377 /* See comment in check_kcov_mode(). */ 378 barrier(); 379 WRITE_ONCE(t->kcov_mode, kcov->mode); 380 t->kcov = kcov; 381 kcov->t = t; 382 /* This is put either in kcov_task_exit() or in KCOV_DISABLE. */ 383 kcov_get(kcov); 384 return 0; 385 case KCOV_DISABLE: 386 /* Disable coverage for the current task. */ 387 unused = arg; 388 if (unused != 0 || current->kcov != kcov) 389 return -EINVAL; 390 t = current; 391 if (WARN_ON(kcov->t != t)) 392 return -EINVAL; 393 kcov_task_init(t); 394 kcov->t = NULL; 395 kcov->mode = KCOV_MODE_INIT; 396 kcov_put(kcov); 397 return 0; 398 default: 399 return -ENOTTY; 400 } 401 } 402 403 static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 404 { 405 struct kcov *kcov; 406 int res; 407 408 kcov = filep->private_data; 409 spin_lock(&kcov->lock); 410 res = kcov_ioctl_locked(kcov, cmd, arg); 411 spin_unlock(&kcov->lock); 412 return res; 413 } 414 415 static const struct file_operations kcov_fops = { 416 .open = kcov_open, 417 .unlocked_ioctl = kcov_ioctl, 418 .compat_ioctl = kcov_ioctl, 419 .mmap = kcov_mmap, 420 .release = kcov_close, 421 }; 422 423 static int __init kcov_init(void) 424 { 425 /* 426 * The kcov debugfs file won't ever get removed and thus, 427 * there is no need to protect it against removal races. The 428 * use of debugfs_create_file_unsafe() is actually safe here. 429 */ 430 if (!debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops)) { 431 pr_err("failed to create kcov in debugfs\n"); 432 return -ENOMEM; 433 } 434 return 0; 435 } 436 437 device_initcall(kcov_init); 438