250 static inline unsigned long metadata_to_pageaddr(const struct kfence_metadata *meta)  in metadata_to_pageaddr()  argument
252 	unsigned long offset = (meta - kfence_metadata + 1) * PAGE_SIZE * 2;  in metadata_to_pageaddr()
258 	if (KFENCE_WARN_ON(meta < kfence_metadata ||  in metadata_to_pageaddr()
259 			   meta >= kfence_metadata + CONFIG_KFENCE_NUM_OBJECTS))  in metadata_to_pageaddr()
266 	if (KFENCE_WARN_ON(ALIGN_DOWN(meta->addr, PAGE_SIZE) != pageaddr))  in metadata_to_pageaddr()
277 metadata_update_state(struct kfence_metadata *meta, enum kfence_object_state next,  in metadata_update_state()  argument
281 		next == KFENCE_OBJECT_FREED ? &meta->free_track : &meta->alloc_track;  in metadata_update_state()
283 	lockdep_assert_held(&meta->lock);  in metadata_update_state()
305 	WRITE_ONCE(meta->state, next);  in metadata_update_state()
311 	struct kfence_metadata *meta;  in check_canary_byte()  local
319 	meta = addr_to_metadata((unsigned long)addr);  in check_canary_byte()
320 	raw_spin_lock_irqsave(&meta->lock, flags);  in check_canary_byte()
321 	kfence_report_error((unsigned long)addr, false, NULL, meta, KFENCE_ERROR_CORRUPTION);  in check_canary_byte()
322 	raw_spin_unlock_irqrestore(&meta->lock, flags);  in check_canary_byte()
327 static inline void set_canary(const struct kfence_metadata *meta)  in set_canary()  argument
329 	const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE);  in set_canary()
336 	for (; addr < meta->addr; addr += sizeof(u64))  in set_canary()
339 	addr = ALIGN_DOWN(meta->addr + meta->size, sizeof(u64));  in set_canary()
344 static inline void check_canary(const struct kfence_metadata *meta)  in check_canary()  argument
346 	const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE);  in check_canary()
359 	for (; meta->addr - addr >= sizeof(u64); addr += sizeof(u64)) {  in check_canary()
369 	for (; addr < meta->addr; addr++) {  in check_canary()
375 	for (addr = meta->addr + meta->size; addr % sizeof(u64) != 0; addr++) {  in check_canary()
394 	struct kfence_metadata *meta = NULL;  in kfence_guarded_alloc()  local
405 		meta = list_entry(kfence_freelist.next, struct kfence_metadata, list);  in kfence_guarded_alloc()
406 		list_del_init(&meta->list);  in kfence_guarded_alloc()
409 	if (!meta) {  in kfence_guarded_alloc()
414 	if (unlikely(!raw_spin_trylock_irqsave(&meta->lock, flags))) {  in kfence_guarded_alloc()
426 		list_add_tail(&meta->list, &kfence_freelist);  in kfence_guarded_alloc()
432 	meta->addr = metadata_to_pageaddr(meta);  in kfence_guarded_alloc()
434 	if (meta->state == KFENCE_OBJECT_FREED)  in kfence_guarded_alloc()
435 		kfence_unprotect(meta->addr);  in kfence_guarded_alloc()
447 		meta->addr += PAGE_SIZE - size;  in kfence_guarded_alloc()
448 		meta->addr = ALIGN_DOWN(meta->addr, cache->align);  in kfence_guarded_alloc()
451 	addr = (void *)meta->addr;  in kfence_guarded_alloc()
454 	metadata_update_state(meta, KFENCE_OBJECT_ALLOCATED, stack_entries, num_stack_entries);  in kfence_guarded_alloc()
456 	WRITE_ONCE(meta->cache, cache);  in kfence_guarded_alloc()
457 	meta->size = size;  in kfence_guarded_alloc()
458 	meta->alloc_stack_hash = alloc_stack_hash;  in kfence_guarded_alloc()
459 	raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_guarded_alloc()
464 	slab = virt_to_slab((void *)meta->addr);  in kfence_guarded_alloc()
473 	set_canary(meta);  in kfence_guarded_alloc()
486 		kfence_protect(meta->addr); /* Random "faults" by protecting the object. */  in kfence_guarded_alloc()
494 static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool zombie)  in kfence_guarded_free()  argument
500 	raw_spin_lock_irqsave(&meta->lock, flags);  in kfence_guarded_free()
502 	if (meta->state != KFENCE_OBJECT_ALLOCATED || meta->addr != (unsigned long)addr) {  in kfence_guarded_free()
505 		kfence_report_error((unsigned long)addr, false, NULL, meta,  in kfence_guarded_free()
507 		raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_guarded_free()
520 	if (meta->unprotected_page) {  in kfence_guarded_free()
521 		memzero_explicit((void *)ALIGN_DOWN(meta->unprotected_page, PAGE_SIZE), PAGE_SIZE);  in kfence_guarded_free()
522 		kfence_protect(meta->unprotected_page);  in kfence_guarded_free()
523 		meta->unprotected_page = 0;  in kfence_guarded_free()
527 	metadata_update_state(meta, KFENCE_OBJECT_FREED, NULL, 0);  in kfence_guarded_free()
528 	init = slab_want_init_on_free(meta->cache);  in kfence_guarded_free()
529 	raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_guarded_free()
531 	alloc_covered_add(meta->alloc_stack_hash, -1);  in kfence_guarded_free()
534 	check_canary(meta);  in kfence_guarded_free()
542 		memzero_explicit(addr, meta->size);  in kfence_guarded_free()
551 		KFENCE_WARN_ON(!list_empty(&meta->list));  in kfence_guarded_free()
552 		list_add_tail(&meta->list, &kfence_freelist);  in kfence_guarded_free()
565 	struct kfence_metadata *meta = container_of(h, struct kfence_metadata, rcu_head);  in rcu_guarded_free()  local
567 	kfence_guarded_free((void *)meta->addr, meta, false);  in rcu_guarded_free()
622 		struct kfence_metadata *meta = &kfence_metadata_init[i];  in kfence_init_pool()  local
625 		INIT_LIST_HEAD(&meta->list);  in kfence_init_pool()
626 		raw_spin_lock_init(&meta->lock);  in kfence_init_pool()
627 		meta->state = KFENCE_OBJECT_UNUSED;  in kfence_init_pool()
628 		meta->addr = addr; /* Initialize for validation in metadata_to_pageaddr(). */  in kfence_init_pool()
629 		list_add_tail(&meta->list, &kfence_freelist);  in kfence_init_pool()
737 	struct kfence_metadata *meta = &kfence_metadata[(long)v - 1];  in show_object()  local
740 	raw_spin_lock_irqsave(&meta->lock, flags);  in show_object()
741 	kfence_print_object(seq, meta);  in show_object()
742 	raw_spin_unlock_irqrestore(&meta->lock, flags);  in show_object()
778 		struct kfence_metadata *meta = &kfence_metadata[i];  in kfence_check_all_canary()  local
780 		if (meta->state == KFENCE_OBJECT_ALLOCATED)  in kfence_check_all_canary()
781 			check_canary(meta);  in kfence_check_all_canary()
987 	struct kfence_metadata *meta;  in kfence_shutdown_cache()  local
997 		meta = &kfence_metadata[i];  in kfence_shutdown_cache()
1006 		if (READ_ONCE(meta->cache) != s ||  in kfence_shutdown_cache()
1007 		    READ_ONCE(meta->state) != KFENCE_OBJECT_ALLOCATED)  in kfence_shutdown_cache()
1010 		raw_spin_lock_irqsave(&meta->lock, flags);  in kfence_shutdown_cache()
1011 		in_use = meta->cache == s && meta->state == KFENCE_OBJECT_ALLOCATED;  in kfence_shutdown_cache()
1012 		raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_shutdown_cache()
1029 			kfence_guarded_free((void *)meta->addr, meta, /*zombie=*/true);  in kfence_shutdown_cache()
1034 		meta = &kfence_metadata[i];  in kfence_shutdown_cache()
1037 		if (READ_ONCE(meta->cache) != s || READ_ONCE(meta->state) != KFENCE_OBJECT_FREED)  in kfence_shutdown_cache()
1040 		raw_spin_lock_irqsave(&meta->lock, flags);  in kfence_shutdown_cache()
1041 		if (meta->cache == s && meta->state == KFENCE_OBJECT_FREED)  in kfence_shutdown_cache()
1042 			meta->cache = NULL;  in kfence_shutdown_cache()
1043 		raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_shutdown_cache()
1122 	const struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);  in kfence_ksize()  local
1128 	return meta ? meta->size : 0;  in kfence_ksize()
1133 	const struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);  in kfence_object_start()  local
1139 	return meta ? (void *)meta->addr : NULL;  in kfence_object_start()
1144 	struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);  in __kfence_free()  local
1147 	KFENCE_WARN_ON(meta->objcg);  in __kfence_free()
1155 	if (unlikely(meta->cache && (meta->cache->flags & SLAB_TYPESAFE_BY_RCU)))  in __kfence_free()
1156 		call_rcu(&meta->rcu_head, rcu_guarded_free);  in __kfence_free()
1158 		kfence_guarded_free(addr, meta, false);  in __kfence_free()
1178 		struct kfence_metadata *meta;  in kfence_handle_page_fault()  local
1181 		meta = addr_to_metadata(addr - PAGE_SIZE);  in kfence_handle_page_fault()
1182 		if (meta && READ_ONCE(meta->state) == KFENCE_OBJECT_ALLOCATED) {  in kfence_handle_page_fault()
1183 			to_report = meta;  in kfence_handle_page_fault()
1185 			distance = addr - data_race(meta->addr + meta->size);  in kfence_handle_page_fault()
1188 		meta = addr_to_metadata(addr + PAGE_SIZE);  in kfence_handle_page_fault()
1189 		if (meta && READ_ONCE(meta->state) == KFENCE_OBJECT_ALLOCATED) {  in kfence_handle_page_fault()
1191 			if (!to_report || distance > data_race(meta->addr) - addr)  in kfence_handle_page_fault()
1192 				to_report = meta;  in kfence_handle_page_fault()