1 #define pr_fmt(fmt) "kcov: " fmt 2 3 #define DISABLE_BRANCH_PROFILING 4 #include <linux/atomic.h> 5 #include <linux/compiler.h> 6 #include <linux/errno.h> 7 #include <linux/export.h> 8 #include <linux/types.h> 9 #include <linux/file.h> 10 #include <linux/fs.h> 11 #include <linux/init.h> 12 #include <linux/mm.h> 13 #include <linux/preempt.h> 14 #include <linux/printk.h> 15 #include <linux/sched.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 #include <linux/vmalloc.h> 19 #include <linux/debugfs.h> 20 #include <linux/uaccess.h> 21 #include <linux/kcov.h> 22 #include <asm/setup.h> 23 24 /* 25 * kcov descriptor (one per opened debugfs file). 26 * State transitions of the descriptor: 27 * - initial state after open() 28 * - then there must be a single ioctl(KCOV_INIT_TRACE) call 29 * - then, mmap() call (several calls are allowed but not useful) 30 * - then, repeated enable/disable for a task (only one task a time allowed) 31 */ 32 struct kcov { 33 /* 34 * Reference counter. We keep one for: 35 * - opened file descriptor 36 * - task with enabled coverage (we can't unwire it from another task) 37 */ 38 atomic_t refcount; 39 /* The lock protects mode, size, area and t. */ 40 spinlock_t lock; 41 enum kcov_mode mode; 42 /* Size of arena (in long's for KCOV_MODE_TRACE). */ 43 unsigned size; 44 /* Coverage buffer shared with user space. */ 45 void *area; 46 /* Task for which we collect coverage, or NULL. */ 47 struct task_struct *t; 48 }; 49 50 /* 51 * Entry point from instrumented code. 52 * This is called once per basic-block/edge. 53 */ 54 void notrace __sanitizer_cov_trace_pc(void) 55 { 56 struct task_struct *t; 57 enum kcov_mode mode; 58 59 t = current; 60 /* 61 * We are interested in code coverage as a function of a syscall inputs, 62 * so we ignore code executed in interrupts. 63 */ 64 if (!t || !in_task()) 65 return; 66 mode = READ_ONCE(t->kcov_mode); 67 if (mode == KCOV_MODE_TRACE) { 68 unsigned long *area; 69 unsigned long pos; 70 unsigned long ip = _RET_IP_; 71 72 #ifdef CONFIG_RANDOMIZE_BASE 73 ip -= kaslr_offset(); 74 #endif 75 76 /* 77 * There is some code that runs in interrupts but for which 78 * in_interrupt() returns false (e.g. preempt_schedule_irq()). 79 * READ_ONCE()/barrier() effectively provides load-acquire wrt 80 * interrupts, there are paired barrier()/WRITE_ONCE() in 81 * kcov_ioctl_locked(). 82 */ 83 barrier(); 84 area = t->kcov_area; 85 /* The first word is number of subsequent PCs. */ 86 pos = READ_ONCE(area[0]) + 1; 87 if (likely(pos < t->kcov_size)) { 88 area[pos] = ip; 89 WRITE_ONCE(area[0], pos); 90 } 91 } 92 } 93 EXPORT_SYMBOL(__sanitizer_cov_trace_pc); 94 95 static void kcov_get(struct kcov *kcov) 96 { 97 atomic_inc(&kcov->refcount); 98 } 99 100 static void kcov_put(struct kcov *kcov) 101 { 102 if (atomic_dec_and_test(&kcov->refcount)) { 103 vfree(kcov->area); 104 kfree(kcov); 105 } 106 } 107 108 void kcov_task_init(struct task_struct *t) 109 { 110 t->kcov_mode = KCOV_MODE_DISABLED; 111 t->kcov_size = 0; 112 t->kcov_area = NULL; 113 t->kcov = NULL; 114 } 115 116 void kcov_task_exit(struct task_struct *t) 117 { 118 struct kcov *kcov; 119 120 kcov = t->kcov; 121 if (kcov == NULL) 122 return; 123 spin_lock(&kcov->lock); 124 if (WARN_ON(kcov->t != t)) { 125 spin_unlock(&kcov->lock); 126 return; 127 } 128 /* Just to not leave dangling references behind. */ 129 kcov_task_init(t); 130 kcov->t = NULL; 131 spin_unlock(&kcov->lock); 132 kcov_put(kcov); 133 } 134 135 static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) 136 { 137 int res = 0; 138 void *area; 139 struct kcov *kcov = vma->vm_file->private_data; 140 unsigned long size, off; 141 struct page *page; 142 143 area = vmalloc_user(vma->vm_end - vma->vm_start); 144 if (!area) 145 return -ENOMEM; 146 147 spin_lock(&kcov->lock); 148 size = kcov->size * sizeof(unsigned long); 149 if (kcov->mode == KCOV_MODE_DISABLED || vma->vm_pgoff != 0 || 150 vma->vm_end - vma->vm_start != size) { 151 res = -EINVAL; 152 goto exit; 153 } 154 if (!kcov->area) { 155 kcov->area = area; 156 vma->vm_flags |= VM_DONTEXPAND; 157 spin_unlock(&kcov->lock); 158 for (off = 0; off < size; off += PAGE_SIZE) { 159 page = vmalloc_to_page(kcov->area + off); 160 if (vm_insert_page(vma, vma->vm_start + off, page)) 161 WARN_ONCE(1, "vm_insert_page() failed"); 162 } 163 return 0; 164 } 165 exit: 166 spin_unlock(&kcov->lock); 167 vfree(area); 168 return res; 169 } 170 171 static int kcov_open(struct inode *inode, struct file *filep) 172 { 173 struct kcov *kcov; 174 175 kcov = kzalloc(sizeof(*kcov), GFP_KERNEL); 176 if (!kcov) 177 return -ENOMEM; 178 atomic_set(&kcov->refcount, 1); 179 spin_lock_init(&kcov->lock); 180 filep->private_data = kcov; 181 return nonseekable_open(inode, filep); 182 } 183 184 static int kcov_close(struct inode *inode, struct file *filep) 185 { 186 kcov_put(filep->private_data); 187 return 0; 188 } 189 190 static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, 191 unsigned long arg) 192 { 193 struct task_struct *t; 194 unsigned long size, unused; 195 196 switch (cmd) { 197 case KCOV_INIT_TRACE: 198 /* 199 * Enable kcov in trace mode and setup buffer size. 200 * Must happen before anything else. 201 */ 202 if (kcov->mode != KCOV_MODE_DISABLED) 203 return -EBUSY; 204 /* 205 * Size must be at least 2 to hold current position and one PC. 206 * Later we allocate size * sizeof(unsigned long) memory, 207 * that must not overflow. 208 */ 209 size = arg; 210 if (size < 2 || size > INT_MAX / sizeof(unsigned long)) 211 return -EINVAL; 212 kcov->size = size; 213 kcov->mode = KCOV_MODE_TRACE; 214 return 0; 215 case KCOV_ENABLE: 216 /* 217 * Enable coverage for the current task. 218 * At this point user must have been enabled trace mode, 219 * and mmapped the file. Coverage collection is disabled only 220 * at task exit or voluntary by KCOV_DISABLE. After that it can 221 * be enabled for another task. 222 */ 223 unused = arg; 224 if (unused != 0 || kcov->mode == KCOV_MODE_DISABLED || 225 kcov->area == NULL) 226 return -EINVAL; 227 if (kcov->t != NULL) 228 return -EBUSY; 229 t = current; 230 /* Cache in task struct for performance. */ 231 t->kcov_size = kcov->size; 232 t->kcov_area = kcov->area; 233 /* See comment in __sanitizer_cov_trace_pc(). */ 234 barrier(); 235 WRITE_ONCE(t->kcov_mode, kcov->mode); 236 t->kcov = kcov; 237 kcov->t = t; 238 /* This is put either in kcov_task_exit() or in KCOV_DISABLE. */ 239 kcov_get(kcov); 240 return 0; 241 case KCOV_DISABLE: 242 /* Disable coverage for the current task. */ 243 unused = arg; 244 if (unused != 0 || current->kcov != kcov) 245 return -EINVAL; 246 t = current; 247 if (WARN_ON(kcov->t != t)) 248 return -EINVAL; 249 kcov_task_init(t); 250 kcov->t = NULL; 251 kcov_put(kcov); 252 return 0; 253 default: 254 return -ENOTTY; 255 } 256 } 257 258 static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 259 { 260 struct kcov *kcov; 261 int res; 262 263 kcov = filep->private_data; 264 spin_lock(&kcov->lock); 265 res = kcov_ioctl_locked(kcov, cmd, arg); 266 spin_unlock(&kcov->lock); 267 return res; 268 } 269 270 static const struct file_operations kcov_fops = { 271 .open = kcov_open, 272 .unlocked_ioctl = kcov_ioctl, 273 .mmap = kcov_mmap, 274 .release = kcov_close, 275 }; 276 277 static int __init kcov_init(void) 278 { 279 /* 280 * The kcov debugfs file won't ever get removed and thus, 281 * there is no need to protect it against removal races. The 282 * use of debugfs_create_file_unsafe() is actually safe here. 283 */ 284 if (!debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops)) { 285 pr_err("failed to create kcov in debugfs\n"); 286 return -ENOMEM; 287 } 288 return 0; 289 } 290 291 device_initcall(kcov_init); 292