Lines Matching refs:meta
250 static inline unsigned long metadata_to_pageaddr(const struct kfence_metadata *meta) in metadata_to_pageaddr() argument
252 unsigned long offset = (meta - kfence_metadata + 1) * PAGE_SIZE * 2; in metadata_to_pageaddr()
258 if (KFENCE_WARN_ON(meta < kfence_metadata || in metadata_to_pageaddr()
259 meta >= kfence_metadata + CONFIG_KFENCE_NUM_OBJECTS)) in metadata_to_pageaddr()
266 if (KFENCE_WARN_ON(ALIGN_DOWN(meta->addr, PAGE_SIZE) != pageaddr)) in metadata_to_pageaddr()
277 metadata_update_state(struct kfence_metadata *meta, enum kfence_object_state next, in metadata_update_state() argument
281 next == KFENCE_OBJECT_FREED ? &meta->free_track : &meta->alloc_track; in metadata_update_state()
283 lockdep_assert_held(&meta->lock); in metadata_update_state()
305 WRITE_ONCE(meta->state, next); in metadata_update_state()
311 struct kfence_metadata *meta; in check_canary_byte() local
319 meta = addr_to_metadata((unsigned long)addr); in check_canary_byte()
320 raw_spin_lock_irqsave(&meta->lock, flags); in check_canary_byte()
321 kfence_report_error((unsigned long)addr, false, NULL, meta, KFENCE_ERROR_CORRUPTION); in check_canary_byte()
322 raw_spin_unlock_irqrestore(&meta->lock, flags); in check_canary_byte()
327 static inline void set_canary(const struct kfence_metadata *meta) in set_canary() argument
329 const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE); in set_canary()
336 for (; addr < meta->addr; addr += sizeof(u64)) in set_canary()
339 addr = ALIGN_DOWN(meta->addr + meta->size, sizeof(u64)); in set_canary()
344 static inline void check_canary(const struct kfence_metadata *meta) in check_canary() argument
346 const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE); in check_canary()
359 for (; meta->addr - addr >= sizeof(u64); addr += sizeof(u64)) { in check_canary()
369 for (; addr < meta->addr; addr++) { in check_canary()
375 for (addr = meta->addr + meta->size; addr % sizeof(u64) != 0; addr++) { in check_canary()
394 struct kfence_metadata *meta = NULL; in kfence_guarded_alloc() local
405 meta = list_entry(kfence_freelist.next, struct kfence_metadata, list); in kfence_guarded_alloc()
406 list_del_init(&meta->list); in kfence_guarded_alloc()
409 if (!meta) { in kfence_guarded_alloc()
414 if (unlikely(!raw_spin_trylock_irqsave(&meta->lock, flags))) { in kfence_guarded_alloc()
426 list_add_tail(&meta->list, &kfence_freelist); in kfence_guarded_alloc()
432 meta->addr = metadata_to_pageaddr(meta); in kfence_guarded_alloc()
434 if (meta->state == KFENCE_OBJECT_FREED) in kfence_guarded_alloc()
435 kfence_unprotect(meta->addr); in kfence_guarded_alloc()
447 meta->addr += PAGE_SIZE - size; in kfence_guarded_alloc()
448 meta->addr = ALIGN_DOWN(meta->addr, cache->align); in kfence_guarded_alloc()
451 addr = (void *)meta->addr; in kfence_guarded_alloc()
454 metadata_update_state(meta, KFENCE_OBJECT_ALLOCATED, stack_entries, num_stack_entries); in kfence_guarded_alloc()
456 WRITE_ONCE(meta->cache, cache); in kfence_guarded_alloc()
457 meta->size = size; in kfence_guarded_alloc()
458 meta->alloc_stack_hash = alloc_stack_hash; in kfence_guarded_alloc()
459 raw_spin_unlock_irqrestore(&meta->lock, flags); in kfence_guarded_alloc()
464 slab = virt_to_slab((void *)meta->addr); in kfence_guarded_alloc()
473 set_canary(meta); in kfence_guarded_alloc()
486 kfence_protect(meta->addr); /* Random "faults" by protecting the object. */ in kfence_guarded_alloc()
494 static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool zombie) in kfence_guarded_free() argument
500 raw_spin_lock_irqsave(&meta->lock, flags); in kfence_guarded_free()
502 if (meta->state != KFENCE_OBJECT_ALLOCATED || meta->addr != (unsigned long)addr) { in kfence_guarded_free()
505 kfence_report_error((unsigned long)addr, false, NULL, meta, in kfence_guarded_free()
507 raw_spin_unlock_irqrestore(&meta->lock, flags); in kfence_guarded_free()
520 if (meta->unprotected_page) { in kfence_guarded_free()
521 memzero_explicit((void *)ALIGN_DOWN(meta->unprotected_page, PAGE_SIZE), PAGE_SIZE); in kfence_guarded_free()
522 kfence_protect(meta->unprotected_page); in kfence_guarded_free()
523 meta->unprotected_page = 0; in kfence_guarded_free()
527 metadata_update_state(meta, KFENCE_OBJECT_FREED, NULL, 0); in kfence_guarded_free()
528 init = slab_want_init_on_free(meta->cache); in kfence_guarded_free()
529 raw_spin_unlock_irqrestore(&meta->lock, flags); in kfence_guarded_free()
531 alloc_covered_add(meta->alloc_stack_hash, -1); in kfence_guarded_free()
534 check_canary(meta); in kfence_guarded_free()
542 memzero_explicit(addr, meta->size); in kfence_guarded_free()
551 KFENCE_WARN_ON(!list_empty(&meta->list)); in kfence_guarded_free()
552 list_add_tail(&meta->list, &kfence_freelist); in kfence_guarded_free()
565 struct kfence_metadata *meta = container_of(h, struct kfence_metadata, rcu_head); in rcu_guarded_free() local
567 kfence_guarded_free((void *)meta->addr, meta, false); in rcu_guarded_free()
622 struct kfence_metadata *meta = &kfence_metadata_init[i]; in kfence_init_pool() local
625 INIT_LIST_HEAD(&meta->list); in kfence_init_pool()
626 raw_spin_lock_init(&meta->lock); in kfence_init_pool()
627 meta->state = KFENCE_OBJECT_UNUSED; in kfence_init_pool()
628 meta->addr = addr; /* Initialize for validation in metadata_to_pageaddr(). */ in kfence_init_pool()
629 list_add_tail(&meta->list, &kfence_freelist); in kfence_init_pool()
737 struct kfence_metadata *meta = &kfence_metadata[(long)v - 1]; in show_object() local
740 raw_spin_lock_irqsave(&meta->lock, flags); in show_object()
741 kfence_print_object(seq, meta); in show_object()
742 raw_spin_unlock_irqrestore(&meta->lock, flags); in show_object()
778 struct kfence_metadata *meta = &kfence_metadata[i]; in kfence_check_all_canary() local
780 if (meta->state == KFENCE_OBJECT_ALLOCATED) in kfence_check_all_canary()
781 check_canary(meta); in kfence_check_all_canary()
987 struct kfence_metadata *meta; in kfence_shutdown_cache() local
997 meta = &kfence_metadata[i]; in kfence_shutdown_cache()
1006 if (READ_ONCE(meta->cache) != s || in kfence_shutdown_cache()
1007 READ_ONCE(meta->state) != KFENCE_OBJECT_ALLOCATED) in kfence_shutdown_cache()
1010 raw_spin_lock_irqsave(&meta->lock, flags); in kfence_shutdown_cache()
1011 in_use = meta->cache == s && meta->state == KFENCE_OBJECT_ALLOCATED; in kfence_shutdown_cache()
1012 raw_spin_unlock_irqrestore(&meta->lock, flags); in kfence_shutdown_cache()
1029 kfence_guarded_free((void *)meta->addr, meta, /*zombie=*/true); in kfence_shutdown_cache()
1034 meta = &kfence_metadata[i]; in kfence_shutdown_cache()
1037 if (READ_ONCE(meta->cache) != s || READ_ONCE(meta->state) != KFENCE_OBJECT_FREED) in kfence_shutdown_cache()
1040 raw_spin_lock_irqsave(&meta->lock, flags); in kfence_shutdown_cache()
1041 if (meta->cache == s && meta->state == KFENCE_OBJECT_FREED) in kfence_shutdown_cache()
1042 meta->cache = NULL; in kfence_shutdown_cache()
1043 raw_spin_unlock_irqrestore(&meta->lock, flags); in kfence_shutdown_cache()
1122 const struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); in kfence_ksize() local
1128 return meta ? meta->size : 0; in kfence_ksize()
1133 const struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); in kfence_object_start() local
1139 return meta ? (void *)meta->addr : NULL; in kfence_object_start()
1144 struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); in __kfence_free() local
1147 KFENCE_WARN_ON(meta->objcg); in __kfence_free()
1155 if (unlikely(meta->cache && (meta->cache->flags & SLAB_TYPESAFE_BY_RCU))) in __kfence_free()
1156 call_rcu(&meta->rcu_head, rcu_guarded_free); in __kfence_free()
1158 kfence_guarded_free(addr, meta, false); in __kfence_free()
1178 struct kfence_metadata *meta; in kfence_handle_page_fault() local
1181 meta = addr_to_metadata(addr - PAGE_SIZE); in kfence_handle_page_fault()
1182 if (meta && READ_ONCE(meta->state) == KFENCE_OBJECT_ALLOCATED) { in kfence_handle_page_fault()
1183 to_report = meta; in kfence_handle_page_fault()
1185 distance = addr - data_race(meta->addr + meta->size); in kfence_handle_page_fault()
1188 meta = addr_to_metadata(addr + PAGE_SIZE); in kfence_handle_page_fault()
1189 if (meta && READ_ONCE(meta->state) == KFENCE_OBJECT_ALLOCATED) { in kfence_handle_page_fault()
1191 if (!to_report || distance > data_race(meta->addr) - addr) in kfence_handle_page_fault()
1192 to_report = meta; in kfence_handle_page_fault()