kfence: save freeing stack trace at calling time instead of freeing time

+29 -10

mm/kfence/core.c

··· 273 273 return pageaddr; 274 274 } 275 275 276 + static inline bool kfence_obj_allocated(const struct kfence_metadata *meta) 277 + { 278 + enum kfence_object_state state = READ_ONCE(meta->state); 279 + 280 + return state == KFENCE_OBJECT_ALLOCATED || state == KFENCE_OBJECT_RCU_FREEING; 281 + } 282 + 276 283 /* 277 284 * Update the object's metadata state, including updating the alloc/free stacks 278 285 * depending on the state transition. ··· 289 282 unsigned long *stack_entries, size_t num_stack_entries) 290 283 { 291 284 struct kfence_track *track = 292 - next == KFENCE_OBJECT_FREED ? &meta->free_track : &meta->alloc_track; 285 + next == KFENCE_OBJECT_ALLOCATED ? &meta->alloc_track : &meta->free_track; 293 286 294 287 lockdep_assert_held(&meta->lock); 288 + 289 + /* Stack has been saved when calling rcu, skip. */ 290 + if (READ_ONCE(meta->state) == KFENCE_OBJECT_RCU_FREEING) 291 + goto out; 295 292 296 293 if (stack_entries) { 297 294 memcpy(track->stack_entries, stack_entries, ··· 312 301 track->cpu = raw_smp_processor_id(); 313 302 track->ts_nsec = local_clock(); /* Same source as printk timestamps. */ 314 303 304 + out: 315 305 /* 316 306 * Pairs with READ_ONCE() in 317 307 * kfence_shutdown_cache(), ··· 518 506 519 507 raw_spin_lock_irqsave(&meta->lock, flags); 520 508 521 - if (meta->state != KFENCE_OBJECT_ALLOCATED || meta->addr != (unsigned long)addr) { 509 + if (!kfence_obj_allocated(meta) || meta->addr != (unsigned long)addr) { 522 510 /* Invalid or double-free, bail out. */ 523 511 atomic_long_inc(&counters[KFENCE_COUNTER_BUGS]); 524 512 kfence_report_error((unsigned long)addr, false, NULL, meta, ··· 796 784 for (i = 0; i < CONFIG_KFENCE_NUM_OBJECTS; i++) { 797 785 struct kfence_metadata *meta = &kfence_metadata[i]; 798 786 799 - if (meta->state == KFENCE_OBJECT_ALLOCATED) 787 + if (kfence_obj_allocated(meta)) 800 788 check_canary(meta); 801 789 } 802 790 } ··· 1022 1010 * the lock will not help, as different critical section 1023 1011 * serialization will have the same outcome. 1024 1012 */ 1025 - if (READ_ONCE(meta->cache) != s || 1026 - READ_ONCE(meta->state) != KFENCE_OBJECT_ALLOCATED) 1013 + if (READ_ONCE(meta->cache) != s || !kfence_obj_allocated(meta)) 1027 1014 continue; 1028 1015 1029 1016 raw_spin_lock_irqsave(&meta->lock, flags); 1030 - in_use = meta->cache == s && meta->state == KFENCE_OBJECT_ALLOCATED; 1017 + in_use = meta->cache == s && kfence_obj_allocated(meta); 1031 1018 raw_spin_unlock_irqrestore(&meta->lock, flags); 1032 1019 1033 1020 if (in_use) { ··· 1171 1160 * the object, as the object page may be recycled for other-typed 1172 1161 * objects once it has been freed. meta->cache may be NULL if the cache 1173 1162 * was destroyed. 1163 + * Save the stack trace here so that reports show where the user freed 1164 + * the object. 1174 1165 */ 1175 - if (unlikely(meta->cache && (meta->cache->flags & SLAB_TYPESAFE_BY_RCU))) 1166 + if (unlikely(meta->cache && (meta->cache->flags & SLAB_TYPESAFE_BY_RCU))) { 1167 + unsigned long flags; 1168 + 1169 + raw_spin_lock_irqsave(&meta->lock, flags); 1170 + metadata_update_state(meta, KFENCE_OBJECT_RCU_FREEING, NULL, 0); 1171 + raw_spin_unlock_irqrestore(&meta->lock, flags); 1176 1172 call_rcu(&meta->rcu_head, rcu_guarded_free); 1177 - else 1173 + } else { 1178 1174 kfence_guarded_free(addr, meta, false); 1175 + } 1179 1176 } 1180 1177 1181 1178 bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs) ··· 1207 1188 int distance = 0; 1208 1189 1209 1190 meta = addr_to_metadata(addr - PAGE_SIZE); 1210 - if (meta && READ_ONCE(meta->state) == KFENCE_OBJECT_ALLOCATED) { 1191 + if (meta && kfence_obj_allocated(meta)) { 1211 1192 to_report = meta; 1212 1193 /* Data race ok; distance calculation approximate. */ 1213 1194 distance = addr - data_race(meta->addr + meta->size); 1214 1195 } 1215 1196 1216 1197 meta = addr_to_metadata(addr + PAGE_SIZE); 1217 - if (meta && READ_ONCE(meta->state) == KFENCE_OBJECT_ALLOCATED) { 1198 + if (meta && kfence_obj_allocated(meta)) { 1218 1199 /* Data race ok; distance calculation approximate. */ 1219 1200 if (!to_report || distance > data_race(meta->addr) - addr) 1220 1201 to_report = meta;

+1

mm/kfence/kfence.h

··· 38 38 enum kfence_object_state { 39 39 KFENCE_OBJECT_UNUSED, /* Object is unused. */ 40 40 KFENCE_OBJECT_ALLOCATED, /* Object is currently allocated. */ 41 + KFENCE_OBJECT_RCU_FREEING, /* Object was allocated, and then being freed by rcu. */ 41 42 KFENCE_OBJECT_FREED, /* Object was allocated, and then freed. */ 42 43 }; 43 44

+4 -3

mm/kfence/report.c

··· 114 114 115 115 /* Timestamp matches printk timestamp format. */ 116 116 seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus (%lu.%06lus ago):\n", 117 - show_alloc ? "allocated" : "freed", track->pid, 117 + show_alloc ? "allocated" : meta->state == KFENCE_OBJECT_RCU_FREEING ? 118 + "rcu freeing" : "freed", track->pid, 118 119 track->cpu, (unsigned long)ts_sec, rem_nsec / 1000, 119 120 (unsigned long)interval_nsec, rem_interval_nsec / 1000); 120 121 ··· 150 149 151 150 kfence_print_stack(seq, meta, true); 152 151 153 - if (meta->state == KFENCE_OBJECT_FREED) { 152 + if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) { 154 153 seq_con_printf(seq, "\n"); 155 154 kfence_print_stack(seq, meta, false); 156 155 } ··· 319 318 kpp->kp_slab_cache = meta->cache; 320 319 kpp->kp_objp = (void *)meta->addr; 321 320 kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack); 322 - if (meta->state == KFENCE_OBJECT_FREED) 321 + if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) 323 322 kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack); 324 323 /* get_stack_skipnr() ensures the first entry is outside allocator. */ 325 324 kpp->kp_ret = kpp->kp_stack[0];

Configure Feed

Configure Feed