40  *   Note that the kmemleak_object.use_count is incremented when an object is
48  *   scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
50  * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
112 #define MSECS_MIN_AGE		5000	/* minimum object age for reporting */
138  * object->lock. Insertions or deletions from object_list, gray_list or
145 	unsigned int flags;		/* object status flags */
150 	/* object usage count; object freed when use_count == 0 */
159 	/* the total number of pointers found pointing to this object */
163 	/* memory ranges to be scanned inside an object (empty for all) */
173 /* flag set after the first reporting of an unreference object */
175 /* flag set to not scan the object */
177 /* flag set to fully scan the object when scan_area allocation failed */
179 /* flag set for object allocated with physical address */
205 /* search tree for object boundaries */
207 /* search tree for object (with OBJECT_PHYS flag) boundaries */
293  * with the object->lock held.
296 			    struct kmemleak_object *object)  in hex_dump_object()  argument
298 	const u8 *ptr = (const u8 *)object->pointer;  in hex_dump_object()
301 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in hex_dump_object()
305 	len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);  in hex_dump_object()
315  * Object colors, encoded with count and min_count:
316  * - white - orphan object, not enough references to it (count < min_count)
321  * Newly created objects don't have any color assigned (object->count == -1)
324 static bool color_white(const struct kmemleak_object *object)  in color_white()  argument
326 	return object->count != KMEMLEAK_BLACK &&  in color_white()
327 		object->count < object->min_count;  in color_white()
330 static bool color_gray(const struct kmemleak_object *object)  in color_gray()  argument
332 	return object->min_count != KMEMLEAK_BLACK &&  in color_gray()
333 		object->count >= object->min_count;  in color_gray()
341 static bool unreferenced_object(struct kmemleak_object *object)  in unreferenced_object()  argument
343 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&  in unreferenced_object()
344 		time_before_eq(object->jiffies + jiffies_min_age,  in unreferenced_object()
350  * print_unreferenced function must be called with the object->lock held.
353 			       struct kmemleak_object *object)  in print_unreferenced()  argument
358 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);  in print_unreferenced()
360 	nr_entries = stack_depot_fetch(object->trace_handle, &entries);  in print_unreferenced()
361 	warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",  in print_unreferenced()
362 			  object->pointer, object->size);  in print_unreferenced()
364 			   object->comm, object->pid, object->jiffies,  in print_unreferenced()
366 	hex_dump_object(seq, object);  in print_unreferenced()
378  * the object->lock held.
380 static void dump_object_info(struct kmemleak_object *object)  in dump_object_info()  argument
382 	pr_notice("Object 0x%08lx (size %zu):\n",  in dump_object_info()
383 			object->pointer, object->size);  in dump_object_info()
385 			object->comm, object->pid, object->jiffies);  in dump_object_info()
386 	pr_notice("  min_count = %d\n", object->min_count);  in dump_object_info()
387 	pr_notice("  count = %d\n", object->count);  in dump_object_info()
388 	pr_notice("  flags = 0x%x\n", object->flags);  in dump_object_info()
389 	pr_notice("  checksum = %u\n", object->checksum);  in dump_object_info()
391 	if (object->trace_handle)  in dump_object_info()
392 		stack_depot_print(object->trace_handle);  in dump_object_info()
396  * Look-up a memory block metadata (kmemleak_object) in the object search
409 		struct kmemleak_object *object;  in __lookup_object()  local
412 		object = rb_entry(rb, struct kmemleak_object, rb_node);  in __lookup_object()
413 		untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in __lookup_object()
416 			rb = object->rb_node.rb_left;  in __lookup_object()
417 		else if (untagged_objp + object->size <= untagged_ptr)  in __lookup_object()
418 			rb = object->rb_node.rb_right;  in __lookup_object()
420 			return object;  in __lookup_object()
422 			kmemleak_warn("Found object by alias at 0x%08lx\n",  in __lookup_object()
424 			dump_object_info(object);  in __lookup_object()
431 /* Look-up a kmemleak object which allocated with virtual address. */
438  * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
439  * that once an object's use_count reached 0, the RCU freeing was already
440  * registered and the object should no longer be used. This function must be
443 static int get_object(struct kmemleak_object *object)  in get_object()  argument
445 	return atomic_inc_not_zero(&object->use_count);  in get_object()
454 	struct kmemleak_object *object;  in mem_pool_alloc()  local
458 		object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));  in mem_pool_alloc()
459 		if (object)  in mem_pool_alloc()
460 			return object;  in mem_pool_alloc()
465 	object = list_first_entry_or_null(&mem_pool_free_list,  in mem_pool_alloc()
466 					  typeof(*object), object_list);  in mem_pool_alloc()
467 	if (object)  in mem_pool_alloc()
468 		list_del(&object->object_list);  in mem_pool_alloc()
470 		object = &mem_pool[--mem_pool_free_count];  in mem_pool_alloc()
475 	return object;  in mem_pool_alloc()
479  * Return the object to either the slab allocator or the memory pool.
481 static void mem_pool_free(struct kmemleak_object *object)  in mem_pool_free()  argument
485 	if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {  in mem_pool_free()
486 		kmem_cache_free(object_cache, object);  in mem_pool_free()
490 	/* add the object to the memory pool free list */  in mem_pool_free()
492 	list_add(&object->object_list, &mem_pool_free_list);  in mem_pool_free()
503 	struct kmemleak_object *object =  in free_object_rcu()  local
508 	 * code accessing this object, hence no need for locking.  in free_object_rcu()
510 	hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {  in free_object_rcu()
514 	mem_pool_free(object);  in free_object_rcu()
518  * Decrement the object use_count. Once the count is 0, free the object using
524 static void put_object(struct kmemleak_object *object)  in put_object()  argument
526 	if (!atomic_dec_and_test(&object->use_count))  in put_object()
530 	WARN_ON(object->flags & OBJECT_ALLOCATED);  in put_object()
535 	 * came from the memory pool. Free the object directly.  in put_object()
538 		call_rcu(&object->rcu, free_object_rcu);  in put_object()
540 		free_object_rcu(&object->rcu);  in put_object()
544  * Look up an object in the object search tree and increase its use_count.
550 	struct kmemleak_object *object;  in __find_and_get_object()  local
554 	object = __lookup_object(ptr, alias, is_phys);  in __find_and_get_object()
557 	/* check whether the object is still available */  in __find_and_get_object()
558 	if (object && !get_object(object))  in __find_and_get_object()
559 		object = NULL;  in __find_and_get_object()
562 	return object;  in __find_and_get_object()
565 /* Look up and get an object which allocated with virtual address. */
572  * Remove an object from the object_tree_root (or object_phys_tree_root)
576 static void __remove_object(struct kmemleak_object *object)  in __remove_object()  argument
578 	rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?  in __remove_object()
581 	if (!(object->del_state & DELSTATE_NO_DELETE))  in __remove_object()
582 		list_del_rcu(&object->object_list);  in __remove_object()
583 	object->del_state |= DELSTATE_REMOVED;  in __remove_object()
587  * Look up an object in the object search tree and remove it from both
589  * returned object's use_count should be at least 1, as initially set
596 	struct kmemleak_object *object;  in find_and_remove_object()  local
599 	object = __lookup_object(ptr, alias, is_phys);  in find_and_remove_object()
600 	if (object)  in find_and_remove_object()
601 		__remove_object(object);  in find_and_remove_object()
604 	return object;  in find_and_remove_object()
635 	struct kmemleak_object *object, *parent;  in __create_object()  local
640 	object = mem_pool_alloc(gfp);  in __create_object()
641 	if (!object) {  in __create_object()
647 	INIT_LIST_HEAD(&object->object_list);  in __create_object()
648 	INIT_LIST_HEAD(&object->gray_list);  in __create_object()
649 	INIT_HLIST_HEAD(&object->area_list);  in __create_object()
650 	raw_spin_lock_init(&object->lock);  in __create_object()
651 	atomic_set(&object->use_count, 1);  in __create_object()
652 	object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0);  in __create_object()
653 	object->pointer = ptr;  in __create_object()
654 	object->size = kfence_ksize((void *)ptr) ?: size;  in __create_object()
655 	object->excess_ref = 0;  in __create_object()
656 	object->min_count = min_count;  in __create_object()
657 	object->count = 0;			/* white color initially */  in __create_object()
658 	object->jiffies = jiffies;  in __create_object()
659 	object->checksum = 0;  in __create_object()
660 	object->del_state = 0;  in __create_object()
664 		object->pid = 0;  in __create_object()
665 		strncpy(object->comm, "hardirq", sizeof(object->comm));  in __create_object()
667 		object->pid = 0;  in __create_object()
668 		strncpy(object->comm, "softirq", sizeof(object->comm));  in __create_object()
670 		object->pid = current->pid;  in __create_object()
677 		strncpy(object->comm, current->comm, sizeof(object->comm));  in __create_object()
681 	object->trace_handle = set_track_prepare();  in __create_object()
687 	 * Only update min_addr and max_addr with object  in __create_object()
706 			kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",  in __create_object()
713 			kmem_cache_free(object_cache, object);  in __create_object()
717 	rb_link_node(&object->rb_node, rb_parent, link);  in __create_object()
718 	rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root :  in __create_object()
720 	list_add_tail_rcu(&object->object_list, &object_list);  in __create_object()
725 /* Create kmemleak object which allocated with virtual address. */
732 /* Create kmemleak object which allocated with physical address. */
740  * Mark the object as not allocated and schedule RCU freeing via put_object().
742 static void __delete_object(struct kmemleak_object *object)  in __delete_object()  argument
746 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));  in __delete_object()
747 	WARN_ON(atomic_read(&object->use_count) < 1);  in __delete_object()
753 	raw_spin_lock_irqsave(&object->lock, flags);  in __delete_object()
754 	object->flags &= ~OBJECT_ALLOCATED;  in __delete_object()
755 	raw_spin_unlock_irqrestore(&object->lock, flags);  in __delete_object()
756 	put_object(object);  in __delete_object()
765 	struct kmemleak_object *object;  in delete_object_full()  local
767 	object = find_and_remove_object(ptr, 0, false);  in delete_object_full()
768 	if (!object) {  in delete_object_full()
770 		kmemleak_warn("Freeing unknown object at 0x%08lx\n",  in delete_object_full()
775 	__delete_object(object);  in delete_object_full()
785 	struct kmemleak_object *object;  in delete_object_part()  local
788 	object = find_and_remove_object(ptr, 1, is_phys);  in delete_object_part()
789 	if (!object) {  in delete_object_part()
791 		kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",  in delete_object_part()
802 	start = object->pointer;  in delete_object_part()
803 	end = object->pointer + object->size;  in delete_object_part()
805 		__create_object(start, ptr - start, object->min_count,  in delete_object_part()
808 		__create_object(ptr + size, end - ptr - size, object->min_count,  in delete_object_part()
811 	__delete_object(object);  in delete_object_part()
814 static void __paint_it(struct kmemleak_object *object, int color)  in __paint_it()  argument
816 	object->min_count = color;  in __paint_it()
818 		object->flags |= OBJECT_NO_SCAN;  in __paint_it()
821 static void paint_it(struct kmemleak_object *object, int color)  in paint_it()  argument
825 	raw_spin_lock_irqsave(&object->lock, flags);  in paint_it()
826 	__paint_it(object, color);  in paint_it()
827 	raw_spin_unlock_irqrestore(&object->lock, flags);  in paint_it()
832 	struct kmemleak_object *object;  in paint_ptr()  local
834 	object = __find_and_get_object(ptr, 0, is_phys);  in paint_ptr()
835 	if (!object) {  in paint_ptr()
836 		kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",  in paint_ptr()
842 	paint_it(object, color);  in paint_ptr()
843 	put_object(object);  in paint_ptr()
847  * Mark an object permanently as gray-colored so that it can no longer be
856  * Mark the object as black-colored so that it is ignored from scans and
865  * Add a scanning area to the object. If at least one such area is added,
871 	struct kmemleak_object *object;  in add_scan_area()  local
876 	object = find_and_get_object(ptr, 1);  in add_scan_area()
877 	if (!object) {  in add_scan_area()
878 		kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",  in add_scan_area()
884 	untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer);  in add_scan_area()
889 	raw_spin_lock_irqsave(&object->lock, flags);  in add_scan_area()
891 		pr_warn_once("Cannot allocate a scan area, scanning the full object\n");  in add_scan_area()
892 		/* mark the object for full scan to avoid false positives */  in add_scan_area()
893 		object->flags |= OBJECT_FULL_SCAN;  in add_scan_area()
897 		size = untagged_objp + object->size - untagged_ptr;  in add_scan_area()
898 	} else if (untagged_ptr + size > untagged_objp + object->size) {  in add_scan_area()
899 		kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);  in add_scan_area()
900 		dump_object_info(object);  in add_scan_area()
909 	hlist_add_head(&area->node, &object->area_list);  in add_scan_area()
911 	raw_spin_unlock_irqrestore(&object->lock, flags);  in add_scan_area()
912 	put_object(object);  in add_scan_area()
916  * Any surplus references (object already gray) to 'ptr' are passed to
918  * vm_struct may be used as an alternative reference to the vmalloc'ed object
924 	struct kmemleak_object *object;  in object_set_excess_ref()  local
926 	object = find_and_get_object(ptr, 0);  in object_set_excess_ref()
927 	if (!object) {  in object_set_excess_ref()
928 		kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",  in object_set_excess_ref()
933 	raw_spin_lock_irqsave(&object->lock, flags);  in object_set_excess_ref()
934 	object->excess_ref = excess_ref;  in object_set_excess_ref()
935 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_set_excess_ref()
936 	put_object(object);  in object_set_excess_ref()
940  * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
941  * pointer. Such object will not be scanned by kmemleak but references to it
947 	struct kmemleak_object *object;  in object_no_scan()  local
949 	object = find_and_get_object(ptr, 0);  in object_no_scan()
950 	if (!object) {  in object_no_scan()
951 		kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);  in object_no_scan()
955 	raw_spin_lock_irqsave(&object->lock, flags);  in object_no_scan()
956 	object->flags |= OBJECT_NO_SCAN;  in object_no_scan()
957 	raw_spin_unlock_irqrestore(&object->lock, flags);  in object_no_scan()
958 	put_object(object);  in object_no_scan()
962  * kmemleak_alloc - register a newly allocated object
963  * @ptr:	pointer to beginning of the object
964  * @size:	size of the object
965  * @min_count:	minimum number of references to this object. If during memory
967  *		the object is reported as a memory leak. If @min_count is 0,
968  *		the object is never reported as a leak. If @min_count is -1,
969  *		the object is ignored (not scanned and not reported as a leak)
972  * This function is called from the kernel allocators when a new object
986  * kmemleak_alloc_percpu - register a newly allocated __percpu object
987  * @ptr:	__percpu pointer to beginning of the object
988  * @size:	size of the object
991  * This function is called from the kernel percpu allocator when a new object
1013  * kmemleak_vmalloc - register a newly vmalloc'ed object
1015  * @size:	size of the object
1019  * object (memory block) is allocated.
1038  * kmemleak_free - unregister a previously registered object
1039  * @ptr:	pointer to beginning of the object
1041  * This function is called from the kernel allocators when an object (memory
1054  * kmemleak_free_part - partially unregister a previously registered object
1055  * @ptr:	pointer to the beginning or inside the object. This also
1072  * kmemleak_free_percpu - unregister a previously registered __percpu object
1073  * @ptr:	__percpu pointer to beginning of the object
1075  * This function is called from the kernel percpu allocator when an object
1092  * kmemleak_update_trace - update object allocation stack trace
1093  * @ptr:	pointer to beginning of the object
1095  * Override the object allocation stack trace for cases where the actual
1100 	struct kmemleak_object *object;  in kmemleak_update_trace()  local
1108 	object = find_and_get_object((unsigned long)ptr, 1);  in kmemleak_update_trace()
1109 	if (!object) {  in kmemleak_update_trace()
1111 		kmemleak_warn("Updating stack trace for unknown object at %p\n",  in kmemleak_update_trace()
1117 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_update_trace()
1118 	object->trace_handle = set_track_prepare();  in kmemleak_update_trace()
1119 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_update_trace()
1121 	put_object(object);  in kmemleak_update_trace()
1126  * kmemleak_not_leak - mark an allocated object as false positive
1127  * @ptr:	pointer to beginning of the object
1129  * Calling this function on an object will cause the memory block to no longer
1142  * kmemleak_ignore - ignore an allocated object
1143  * @ptr:	pointer to beginning of the object
1145  * Calling this function on an object will cause the memory block to be
1160  * kmemleak_scan_area - limit the range to be scanned in an allocated object
1161  * @ptr:	pointer to beginning or inside the object. This also
1166  * This function is used when it is known that only certain parts of an object
1180  * kmemleak_no_scan - do not scan an allocated object
1181  * @ptr:	pointer to beginning of the object
1184  * in situations where it is known that the given object does not contain any
1200  * @phys:	physical address of the object
1201  * @size:	size of the object
1210 		 * Create object with OBJECT_PHYS flag and  in kmemleak_alloc_phys()
1220  * @phys:	physical address if the beginning or inside an object. This
1236  * @phys:	physical address of the object
1248  * Update an object's checksum and return true if it was modified.
1250 static bool update_checksum(struct kmemleak_object *object)  in update_checksum()  argument
1252 	u32 old_csum = object->checksum;  in update_checksum()
1254 	if (WARN_ON_ONCE(object->flags & OBJECT_PHYS))  in update_checksum()
1259 	object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);  in update_checksum()
1263 	return object->checksum != old_csum;  in update_checksum()
1267  * Update an object's references. object->lock must be held by the caller.
1269 static void update_refs(struct kmemleak_object *object)  in update_refs()  argument
1271 	if (!color_white(object)) {  in update_refs()
1277 	 * Increase the object's reference count (number of pointers to the  in update_refs()
1279 	 * object's color will become gray and it will be added to the  in update_refs()
1282 	object->count++;  in update_refs()
1283 	if (color_gray(object)) {  in update_refs()
1285 		WARN_ON(!get_object(object));  in update_refs()
1286 		list_add_tail(&object->gray_list, &gray_list);  in update_refs()
1326 		struct kmemleak_object *object;  in scan_block()  local
1343 		 * object->use_count cannot be dropped to 0 while the object  in scan_block()
1347 		object = lookup_object(pointer, 1);  in scan_block()
1348 		if (!object)  in scan_block()
1350 		if (object == scanned)  in scan_block()
1355 		 * Avoid the lockdep recursive warning on object->lock being  in scan_block()
1359 		raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1360 		/* only pass surplus references (object already gray) */  in scan_block()
1361 		if (color_gray(object)) {  in scan_block()
1362 			excess_ref = object->excess_ref;  in scan_block()
1363 			/* no need for update_refs() if object already gray */  in scan_block()
1366 			update_refs(object);  in scan_block()
1368 		raw_spin_unlock(&object->lock);  in scan_block()
1371 			object = lookup_object(excess_ref, 0);  in scan_block()
1372 			if (!object)  in scan_block()
1374 			if (object == scanned)  in scan_block()
1377 			raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);  in scan_block()
1378 			update_refs(object);  in scan_block()
1379 			raw_spin_unlock(&object->lock);  in scan_block()
1404  * that object->use_count >= 1.
1406 static void scan_object(struct kmemleak_object *object)  in scan_object()  argument
1413 	 * Once the object->lock is acquired, the corresponding memory block  in scan_object()
1416 	raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1417 	if (object->flags & OBJECT_NO_SCAN)  in scan_object()
1419 	if (!(object->flags & OBJECT_ALLOCATED))  in scan_object()
1420 		/* already freed object */  in scan_object()
1423 	obj_ptr = object->flags & OBJECT_PHYS ?  in scan_object()
1424 		  __va((phys_addr_t)object->pointer) :  in scan_object()
1425 		  (void *)object->pointer;  in scan_object()
1427 	if (hlist_empty(&object->area_list) ||  in scan_object()
1428 	    object->flags & OBJECT_FULL_SCAN) {  in scan_object()
1430 		void *end = obj_ptr + object->size;  in scan_object()
1435 			scan_block(start, next, object);  in scan_object()
1441 			raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1443 			raw_spin_lock_irqsave(&object->lock, flags);  in scan_object()
1444 		} while (object->flags & OBJECT_ALLOCATED);  in scan_object()
1446 		hlist_for_each_entry(area, &object->area_list, node)  in scan_object()
1449 				   object);  in scan_object()
1451 	raw_spin_unlock_irqrestore(&object->lock, flags);  in scan_object()
1460 	struct kmemleak_object *object, *tmp;  in scan_gray_list()  local
1467 	object = list_entry(gray_list.next, typeof(*object), gray_list);  in scan_gray_list()
1468 	while (&object->gray_list != &gray_list) {  in scan_gray_list()
1473 			scan_object(object);  in scan_gray_list()
1475 		tmp = list_entry(object->gray_list.next, typeof(*object),  in scan_gray_list()
1478 		/* remove the object from the list and release it */  in scan_gray_list()
1479 		list_del(&object->gray_list);  in scan_gray_list()
1480 		put_object(object);  in scan_gray_list()
1482 		object = tmp;  in scan_gray_list()
1488  * Conditionally call resched() in an object iteration loop while making sure
1489  * that the given object won't go away without RCU read lock by performing a
1492 static void kmemleak_cond_resched(struct kmemleak_object *object)  in kmemleak_cond_resched()  argument
1494 	if (!get_object(object))  in kmemleak_cond_resched()
1495 		return;	/* Try next object */  in kmemleak_cond_resched()
1498 	if (object->del_state & DELSTATE_REMOVED)  in kmemleak_cond_resched()
1499 		goto unlock_put;	/* Object removed */  in kmemleak_cond_resched()
1500 	object->del_state |= DELSTATE_NO_DELETE;  in kmemleak_cond_resched()
1508 	if (object->del_state & DELSTATE_REMOVED)  in kmemleak_cond_resched()
1509 		list_del_rcu(&object->object_list);  in kmemleak_cond_resched()
1510 	object->del_state &= ~DELSTATE_NO_DELETE;  in kmemleak_cond_resched()
1513 	put_object(object);  in kmemleak_cond_resched()
1523 	struct kmemleak_object *object;  in kmemleak_scan()  local
1532 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1533 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1537 		 * 1 reference to any object at this point.  in kmemleak_scan()
1539 		if (atomic_read(&object->use_count) > 1) {  in kmemleak_scan()
1540 			pr_debug("object->use_count = %d\n",  in kmemleak_scan()
1541 				 atomic_read(&object->use_count));  in kmemleak_scan()
1542 			dump_object_info(object);  in kmemleak_scan()
1547 		if ((object->flags & OBJECT_PHYS) &&  in kmemleak_scan()
1548 		   !(object->flags & OBJECT_NO_SCAN)) {  in kmemleak_scan()
1549 			unsigned long phys = object->pointer;  in kmemleak_scan()
1552 			    PHYS_PFN(phys + object->size) > max_low_pfn)  in kmemleak_scan()
1553 				__paint_it(object, KMEMLEAK_BLACK);  in kmemleak_scan()
1556 		/* reset the reference count (whiten the object) */  in kmemleak_scan()
1557 		object->count = 0;  in kmemleak_scan()
1558 		if (color_gray(object) && get_object(object))  in kmemleak_scan()
1559 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1561 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1564 			kmemleak_cond_resched(object);  in kmemleak_scan()
1632 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1634 			kmemleak_cond_resched(object);  in kmemleak_scan()
1641 		if (!color_white(object))  in kmemleak_scan()
1643 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1644 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)  in kmemleak_scan()
1645 		    && update_checksum(object) && get_object(object)) {  in kmemleak_scan()
1647 			object->count = object->min_count;  in kmemleak_scan()
1648 			list_add_tail(&object->gray_list, &gray_list);  in kmemleak_scan()
1650 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1669 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_scan()
1671 			kmemleak_cond_resched(object);  in kmemleak_scan()
1678 		if (!color_white(object))  in kmemleak_scan()
1680 		raw_spin_lock_irq(&object->lock);  in kmemleak_scan()
1681 		if (unreferenced_object(object) &&  in kmemleak_scan()
1682 		    !(object->flags & OBJECT_REPORTED)) {  in kmemleak_scan()
1683 			object->flags |= OBJECT_REPORTED;  in kmemleak_scan()
1686 				print_unreferenced(NULL, object);  in kmemleak_scan()
1690 		raw_spin_unlock_irq(&object->lock);  in kmemleak_scan()
1768  * Iterate over the object_list and return the first valid object at or after
1774 	struct kmemleak_object *object;  in kmemleak_seq_start()  local
1783 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_seq_start()
1786 		if (get_object(object))  in kmemleak_seq_start()
1789 	object = NULL;  in kmemleak_seq_start()
1791 	return object;  in kmemleak_seq_start()
1795  * Return the next object in the object_list. The function decrements the
1796  * use_count of the previous object and increases that of the next one.
1818  * Decrement the use_count of the last object required, if any.
1835  * Print the information for an unreferenced object to the seq file.
1839 	struct kmemleak_object *object = v;  in kmemleak_seq_show()  local
1842 	raw_spin_lock_irqsave(&object->lock, flags);  in kmemleak_seq_show()
1843 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))  in kmemleak_seq_show()
1844 		print_unreferenced(seq, object);  in kmemleak_seq_show()
1845 	raw_spin_unlock_irqrestore(&object->lock, flags);  in kmemleak_seq_show()
1864 	struct kmemleak_object *object;  in dump_str_object_info()  local
1869 	object = find_and_get_object(addr, 0);  in dump_str_object_info()
1870 	if (!object) {  in dump_str_object_info()
1871 		pr_info("Unknown object at 0x%08lx\n", addr);  in dump_str_object_info()
1875 	raw_spin_lock_irqsave(&object->lock, flags);  in dump_str_object_info()
1876 	dump_object_info(object);  in dump_str_object_info()
1877 	raw_spin_unlock_irqrestore(&object->lock, flags);  in dump_str_object_info()
1879 	put_object(object);  in dump_str_object_info()
1891 	struct kmemleak_object *object;  in kmemleak_clear()  local
1894 	list_for_each_entry_rcu(object, &object_list, object_list) {  in kmemleak_clear()
1895 		raw_spin_lock_irq(&object->lock);  in kmemleak_clear()
1896 		if ((object->flags & OBJECT_REPORTED) &&  in kmemleak_clear()
1897 		    unreferenced_object(object))  in kmemleak_clear()
1898 			__paint_it(object, KMEMLEAK_GREY);  in kmemleak_clear()
1899 		raw_spin_unlock_irq(&object->lock);  in kmemleak_clear()
1922  *   dump=...	- dump information about the object found at the given address
2008 	struct kmemleak_object *object, *tmp;  in __kmemleak_do_cleanup()  local
2014 	list_for_each_entry_safe(object, tmp, &object_list, object_list) {  in __kmemleak_do_cleanup()
2015 		__remove_object(object);  in __kmemleak_do_cleanup()
2016 		__delete_object(object);  in __kmemleak_do_cleanup()
2032 	 * longer track object freeing. Ordering of the scan thread stopping and  in kmemleak_do_cleanup()