···11-#define PRIVATE 1
22-#include <sys/cdefs.h>
33-44-#include <darling/emulation/ext/for-xtrace.h>
55-#include <stdint.h>
66-#include <stdbool.h>
77-#include <string.h>
88-#include <sys/queue.h>
99-1010-#include "malloc.h"
1111-#include "lock.h"
1212-#include <darling/emulation/simple.h>
1313-1414-#ifndef XTRACE_MALLOC_DEBUG
1515- #define XTRACE_MALLOC_DEBUG 0
1616-#endif
1717-1818-#if XTRACE_MALLOC_DEBUG
1919- #define xtrace_malloc_debug(x, ...) xtrace_log(x "\n", ## __VA_ARGS__)
2020- #undef XTRACE_INLINE
2121- #define XTRACE_INLINE
2222-#else
2323- #define xtrace_malloc_debug(x, ...)
2424-#endif
2525-2626-// we can't rely on libmalloc because:
2727-// 1. we should be invisible to everyone but libkernel
2828-// 2. libmalloc might need to `thread_switch(2)`, which will recurse back into xtrace, blowing up the stack
2929-// so let's roll our own malloc! (using Linux's mmap)
3030-3131-// all blocks will be allocated in multiples of this size
3232-#define BLOCK_SIZE_MULTIPLE (4096ULL)
3333-3434-#define LIST_PREV(elm, link) ({ \
3535- __typeof(elm) elm_cache = elm; \
3636- __typeof((elm_cache)->link)* as_link = __container_of(elm_cache->link.le_prev, __typeof((elm_cache)->link), le_next); \
3737- elm_cache = __container_of(as_link, __typeof(*elm_cache), link); \
3838- elm_cache; \
3939- })
4040-4141-typedef LIST_HEAD(xtrace_memory_fragment_head, xtrace_memory_fragment) xtrace_memory_fragment_head_t;
4242-typedef LIST_ENTRY(xtrace_memory_fragment) xtrace_memory_fragment_entry_t;
4343-4444-//
4545-// xtrace_memory_fragment
4646-//
4747-4848-#define XTRACE_MEMORY_FRAGMENT_FLAG_IN_USE (1ULL << 63)
4949-#define XTRACE_MEMORY_FRAGMENT_FLAG_IS_MMAP_BASE (1ULL << 62)
5050-#define XTRACE_MEMORY_FRAGMENT_FLAG_BIT_COUNT (2)
5151-#define XTRACE_MEMORY_FRAGMENT_SIZE_MASK (0xffffffffffffffffULL >> XTRACE_MEMORY_FRAGMENT_FLAG_BIT_COUNT)
5252-5353-typedef struct xtrace_memory_fragment* xtrace_memory_fragment_t;
5454-struct xtrace_memory_fragment {
5555- uint64_t flags;
5656- xtrace_memory_fragment_entry_t block_link;
5757- xtrace_memory_fragment_entry_t free_link;
5858- char memory[];
5959-};
6060-6161-XTRACE_INLINE
6262-bool xtrace_memory_fragment_in_use(xtrace_memory_fragment_t fragment) {
6363- return fragment->flags & XTRACE_MEMORY_FRAGMENT_FLAG_IN_USE;
6464-};
6565-6666-XTRACE_INLINE
6767-void xtrace_memory_fragment_set_in_use(xtrace_memory_fragment_t fragment, bool in_use) {
6868- if (in_use) {
6969- fragment->flags |= XTRACE_MEMORY_FRAGMENT_FLAG_IN_USE;
7070- } else {
7171- fragment->flags &= ~XTRACE_MEMORY_FRAGMENT_FLAG_IN_USE;
7272- }
7373-};
7474-7575-XTRACE_INLINE
7676-bool xtrace_memory_fragment_is_mmap_base(xtrace_memory_fragment_t fragment) {
7777- return fragment->flags & XTRACE_MEMORY_FRAGMENT_FLAG_IS_MMAP_BASE;
7878-};
7979-8080-XTRACE_INLINE
8181-void xtrace_memory_fragment_set_is_mmap_base(xtrace_memory_fragment_t fragment, bool is_mmap_base) {
8282- if (is_mmap_base) {
8383- fragment->flags |= XTRACE_MEMORY_FRAGMENT_FLAG_IS_MMAP_BASE;
8484- } else {
8585- fragment->flags &= ~XTRACE_MEMORY_FRAGMENT_FLAG_IS_MMAP_BASE;
8686- }
8787-};
8888-8989-XTRACE_INLINE
9090-size_t xtrace_memory_fragment_size(xtrace_memory_fragment_t fragment) {
9191- return fragment->flags & XTRACE_MEMORY_FRAGMENT_SIZE_MASK;
9292-};
9393-9494-XTRACE_INLINE
9595-void xtrace_memory_fragment_set_size(xtrace_memory_fragment_t fragment, size_t new_size) {
9696- fragment->flags = (fragment->flags & ~XTRACE_MEMORY_FRAGMENT_SIZE_MASK) | (new_size & XTRACE_MEMORY_FRAGMENT_SIZE_MASK);
9797-};
9898-9999-XTRACE_INLINE
100100-void xtrace_memory_fragment_init(xtrace_memory_fragment_t fragment, size_t size) {
101101- xtrace_malloc_debug("initializing fragment of %llu bytes at %p", size, fragment);
102102- fragment->flags = size & XTRACE_MEMORY_FRAGMENT_SIZE_MASK;
103103- fragment->block_link.le_next = NULL;
104104- fragment->block_link.le_prev = NULL;
105105- fragment->free_link.le_next = NULL;
106106- fragment->free_link.le_prev = NULL;
107107-};
108108-109109-//
110110-// xtrace_memory_block
111111-//
112112-113113-typedef struct xtrace_memory_block* xtrace_memory_block_t;
114114-struct xtrace_memory_block {
115115- uint64_t size;
116116- xtrace_memory_fragment_head_t fragment_list;
117117- struct xtrace_memory_fragment fragment;
118118-};
119119-120120-XTRACE_INLINE
121121-bool xtrace_memory_block_in_use(xtrace_memory_block_t block) {
122122- // the block is in use if either:
123123- // * the first fragment is in use, or
124124- // * the block has been fragmented
125125- return xtrace_memory_fragment_in_use(&block->fragment) || (block->size != xtrace_memory_fragment_size(&block->fragment));
126126-};
127127-128128-XTRACE_INLINE
129129-void xtrace_memory_block_init(xtrace_memory_block_t block, size_t size) {
130130- block->size = size;
131131- LIST_INIT(&block->fragment_list);
132132- xtrace_memory_fragment_init(&block->fragment, block->size);
133133- xtrace_memory_fragment_set_is_mmap_base(&block->fragment, true);
134134- LIST_INSERT_HEAD(&block->fragment_list, &block->fragment, block_link);
135135-};
136136-137137-//
138138-// global variables
139139-//
140140-141141-// list of available fragments
142142-static xtrace_memory_fragment_head_t free_list = LIST_HEAD_INITIALIZER(free_list);
143143-// lock for free_list
144144-static xtrace_lock_t free_lock = XTRACE_LOCK_INITIALIZER;
145145-146146-//
147147-// internal functions
148148-//
149149-150150-static xtrace_memory_fragment_t allocate_fragment(size_t required_size);
151151-static void release_fragment(xtrace_memory_fragment_t fragment);
152152-static bool shrink_fragment(xtrace_memory_fragment_t fragment, size_t new_size);
153153-static bool expand_fragment(xtrace_memory_fragment_t fragment, size_t new_size);
154154-155155-// borrowed from libgmalloc
156156-XTRACE_INLINE
157157-size_t round_up(size_t size, size_t increment) {
158158- if ((size & (increment - 1)) == 0) {
159159- return size;
160160- }
161161- return (size | (increment - 1)) + 1;
162162-}
163163-164164-static xtrace_memory_fragment_t allocate_fragment(size_t required_size) {
165165- xtrace_memory_fragment_t viable_fragment = NULL;
166166- size_t viable_fragment_size = 0;
167167- xtrace_memory_fragment_t loop_var = NULL;
168168- xtrace_lock_lock(&free_lock);
169169-170170- // look for the smallest free fragment
171171- LIST_FOREACH(loop_var, &free_list, free_link) {
172172- size_t loop_var_size = xtrace_memory_fragment_size(loop_var);
173173-174174- // 1. always make sure the current fragment is large enough
175175- // 2. we'll prefer the current fragment over the previous candidate if either:
176176- // * we don't currently have a candidate fragment, or
177177- // * the current fragment is smaller than the previous candidate
178178- if (loop_var_size > required_size && (viable_fragment == NULL || loop_var_size < viable_fragment_size)) {
179179- viable_fragment = loop_var;
180180- viable_fragment_size = loop_var_size;
181181- xtrace_malloc_debug("found viable fragment with address %p and size %llu", viable_fragment->memory, viable_fragment_size);
182182- }
183183- }
184184-185185- // if we didn't find a viable candidate...
186186- if (viable_fragment == NULL) {
187187- // ...allocate some memory
188188- size_t allocation_size = round_up(required_size, BLOCK_SIZE_MULTIPLE);
189189- xtrace_memory_block_t block = (xtrace_memory_block_t)_mmap_for_xtrace(NULL, allocation_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
190190-191191- xtrace_malloc_debug("mmap for %llu bytes returned %p", allocation_size, block);
192192-193193- // check if return value is an error code
194194- // see similar check in libkernel's `mman.c`
195195- if ((unsigned long)block > (unsigned long)-4096) {
196196- // we failed to allocate the memory, we can't do anything else
197197- xtrace_malloc_debug("mmap result was failure");
198198- goto out;
199199- }
200200-201201- // initialize it
202202- xtrace_memory_block_init(block, allocation_size - sizeof(struct xtrace_memory_block));
203203-204204- // add it to the free list
205205- LIST_INSERT_HEAD(&free_list, &block->fragment, free_link);
206206-207207- // and set `viable_fragment` so our code below can handle it the same way as the no-mmap case
208208- viable_fragment = &block->fragment;
209209- }
210210-211211- // at this point, `viable_fragment` is guaranteed to have a value
212212- xtrace_malloc_debug("final fragment for allocation: %llu bytes at %p", xtrace_memory_fragment_size(viable_fragment), viable_fragment->memory);
213213-214214- // shrink the fragment (if possible)
215215- shrink_fragment(viable_fragment, required_size);
216216-217217- // set the in_use bit
218218- xtrace_memory_fragment_set_in_use(viable_fragment, true);
219219-220220- // finally, remove the viable fragment from the free list
221221- LIST_REMOVE(viable_fragment, free_link);
222222-223223-out:
224224- xtrace_lock_unlock(&free_lock);
225225-226226- return viable_fragment;
227227-};
228228-229229-static void release_fragment(xtrace_memory_fragment_t fragment) {
230230- xtrace_lock_lock(&free_lock);
231231-232232- xtrace_malloc_debug("releasing fragment of %llu bytes at %p", xtrace_memory_fragment_size(fragment), fragment->memory);
233233-234234- // clear the in_use bit
235235- xtrace_memory_fragment_set_in_use(fragment, false);
236236-237237- // add it to the free list
238238- LIST_INSERT_HEAD(&free_list, fragment, free_link);
239239-240240- // try to defragment the block
241241- while (true) {
242242- xtrace_memory_fragment_t prev_frag = xtrace_memory_fragment_is_mmap_base(fragment) ? NULL : LIST_PREV(fragment, block_link);
243243- xtrace_memory_fragment_t next_frag = LIST_NEXT(fragment, block_link);
244244- bool defragged = false;
245245-246246- if (prev_frag && !xtrace_memory_fragment_in_use(prev_frag)) {
247247- xtrace_malloc_debug("defragmenting with previous fragment (%llu bytes at %p)", xtrace_memory_fragment_size(prev_frag), prev_frag->memory);
248248- defragged = true;
249249-250250- // remove the current fragment from the free list
251251- LIST_REMOVE(fragment, free_link);
252252-253253- // remove the current fragment from the block list
254254- LIST_REMOVE(fragment, block_link);
255255-256256- // expand the previous fragment
257257- xtrace_memory_fragment_set_size(prev_frag, xtrace_memory_fragment_size(prev_frag) + sizeof(struct xtrace_memory_fragment) + xtrace_memory_fragment_size(fragment));
258258-259259- // `fragment` is now gone; set it to `prev_frag`
260260- fragment = prev_frag;
261261- }
262262-263263- if (next_frag && !xtrace_memory_fragment_in_use(next_frag)) {
264264- xtrace_malloc_debug("defragmenting with next fragment (%llu bytes at %p)", xtrace_memory_fragment_size(fragment), fragment->memory);
265265- defragged = true;
266266-267267- // remove the next fragment from the free list
268268- LIST_REMOVE(next_frag, free_link);
269269-270270- // remove the next fragment from the block list
271271- LIST_REMOVE(next_frag, block_link);
272272-273273- // expand the current fragment
274274- xtrace_memory_fragment_set_size(fragment, xtrace_memory_fragment_size(fragment) + sizeof(struct xtrace_memory_fragment) + xtrace_memory_fragment_size(next_frag));
275275-276276- // `next_frag` is now gone
277277- }
278278-279279- // if we didn't defragment anything on this loop, we can't defragment any more
280280- if (!defragged) {
281281- xtrace_malloc_debug("cannot defragment any further");
282282- break;
283283- } else {
284284- // otherwise, keep looping to try to defragment more
285285- xtrace_malloc_debug("doing another defragmentation iteration");
286286- }
287287- }
288288-289289- xtrace_malloc_debug("final releasing fragment is %llu bytes at %p", xtrace_memory_fragment_size(fragment), fragment->memory);
290290-291291- // if `fragment` is the mmap base fragment...
292292- if (xtrace_memory_fragment_is_mmap_base(fragment)) {
293293- xtrace_memory_block_t block = __container_of(fragment, struct xtrace_memory_block, fragment);
294294- xtrace_malloc_debug("releasing fragment is base fragment");
295295-296296- // ...and we managed to defragment the entire block...
297297- if (!xtrace_memory_block_in_use(block)) {
298298- // ...unmap it!
299299- xtrace_malloc_debug("unmapping completely freed block (%llu bytes at %p)", block->size, block->fragment.memory);
300300-301301- // start off by removing all traces of this block in the free list.
302302- // at this point, only the base fragment should be in the free list, so remove it
303303- LIST_REMOVE(fragment, free_link);
304304-305305- // and now, just unmap it
306306- _munmap_for_xtrace(block, block->size + sizeof(struct xtrace_memory_block));
307307- // and we ignore errors
308308- }
309309- }
310310-311311-out:
312312- xtrace_lock_unlock(&free_lock);
313313-};
314314-315315-// must be called with free_lock held
316316-static bool shrink_fragment(xtrace_memory_fragment_t fragment, size_t new_size) {
317317- xtrace_malloc_debug("shrink to %llu bytes requested for fragment of %llu bytes at %p", new_size, xtrace_memory_fragment_size(fragment), fragment->memory);
318318-319319- // calculate the remaining size if we were to shrink this fragment
320320- size_t remaining_size = xtrace_memory_fragment_size(fragment) - new_size;
321321-322322- // if we would have enough space to create another fragment after shrinking this one...
323323- if (remaining_size > sizeof(struct xtrace_memory_fragment)) {
324324- xtrace_malloc_debug("fragment eligible; shrinking...");
325325-326326- // ...shrink it...
327327- xtrace_memory_fragment_set_size(fragment, new_size);
328328-329329- // ...and create the new fragment
330330- xtrace_memory_fragment_t new_fragment = (xtrace_memory_fragment_t)(fragment->memory + new_size);
331331- xtrace_memory_fragment_init(new_fragment, remaining_size - sizeof(struct xtrace_memory_fragment));
332332-333333- // add it to the block list
334334- LIST_INSERT_AFTER(fragment, new_fragment, block_link);
335335-336336- xtrace_malloc_debug("old fragment shrunk to %llu bytes at %p", xtrace_memory_fragment_size(fragment), fragment->memory);
337337- xtrace_malloc_debug("new fragment of %llu bytes created at %p", xtrace_memory_fragment_size(new_fragment), new_fragment->memory);
338338-339339- // insert it into the free list
340340- LIST_INSERT_HEAD(&free_list, new_fragment, free_link);
341341-342342- return true;
343343- } else {
344344- // otherwise, we don't want to shrink this one to avoid holes in our block
345345- xtrace_malloc_debug("fragment ineligible for shrinking (not enough space after it)");
346346- }
347347-348348- return false;
349349-};
350350-351351-// must be called with free_lock held
352352-static bool expand_fragment(xtrace_memory_fragment_t fragment, size_t new_size) {
353353- xtrace_malloc_debug("expansion to %llu bytes requested for fragment of %llu bytes at %p", new_size, xtrace_memory_fragment_size(fragment), fragment->memory);
354354-355355- size_t available_size = xtrace_memory_fragment_size(fragment);
356356- xtrace_memory_fragment_t loop_var = fragment;
357357- xtrace_memory_fragment_t end_frag = NULL;
358358-359359- while ((end_frag = LIST_NEXT(loop_var, block_link)) != NULL) {
360360- // if the fragment is in use...
361361- if (xtrace_memory_fragment_in_use(end_frag)) {
362362- // ...we've reached a limit
363363- xtrace_malloc_debug("insufficient free fragments for expansion");
364364- break;
365365- }
366366-367367- available_size += xtrace_memory_fragment_size(end_frag) + sizeof(struct xtrace_memory_fragment);
368368- loop_var = end_frag;
369369-370370- // if we've got enough...
371371- if (available_size >= new_size) {
372372- xtrace_malloc_debug("found enough free fragments for expansion");
373373- // ...we're done; we don't need to keep looking for more
374374- break;
375375- }
376376- }
377377-378378- if (end_frag && available_size >= new_size) {
379379- xtrace_memory_fragment_t terminating_value = LIST_NEXT(end_frag, block_link);
380380-381381- while (LIST_NEXT(fragment, block_link) != terminating_value) {
382382- xtrace_memory_fragment_t dying_fragment = LIST_NEXT(fragment, block_link);
383383-384384- xtrace_malloc_debug("absorbing fragment of %llu bytes at %p", xtrace_memory_fragment_size(dying_fragment), dying_fragment->memory);
385385-386386- // remove the dying fragment from the free list
387387- LIST_REMOVE(dying_fragment, free_link);
388388-389389- // remove the dying fragment from the block list
390390- LIST_REMOVE(dying_fragment, block_link);
391391-392392- // expand the surviving fragment
393393- xtrace_memory_fragment_set_size(fragment, xtrace_memory_fragment_size(fragment) + sizeof(struct xtrace_memory_fragment) + xtrace_memory_fragment_size(dying_fragment));
394394-395395- // `dying_fragment` is now gone
396396- }
397397-398398- xtrace_malloc_debug("expanded fragment is %llu bytes at %p; trying to shrink...", xtrace_memory_fragment_size(fragment), fragment->memory);
399399-400400- // try to shrink it so that we leave space for other possible fragments
401401- shrink_fragment(fragment, new_size);
402402-403403- xtrace_malloc_debug("final fragment is %llu bytes at %p", xtrace_memory_fragment_size(fragment), fragment->memory);
404404-405405- return true;
406406- }
407407-408408- xtrace_malloc_debug("ineligible for expansion");
409409-410410- return false;
411411-};
412412-413413-//
414414-// api functions
415415-//
416416-417417-extern "C"
418418-void* xtrace_malloc(size_t size) {
419419- if (size == 0) {
420420- return NULL;
421421- }
422422- xtrace_memory_fragment_t fragment = allocate_fragment(size);
423423- if (fragment == NULL) {
424424- return NULL;
425425- }
426426- return fragment->memory;
427427-};
428428-429429-extern "C"
430430-void xtrace_free(void* pointer) {
431431- if (pointer == NULL) {
432432- return;
433433- }
434434- xtrace_memory_fragment_t fragment = __container_of((const char (*)[])pointer, struct xtrace_memory_fragment, memory);
435435- release_fragment(fragment);
436436-};
437437-438438-extern "C"
439439-void* xtrace_realloc(void* old_pointer, size_t new_size) {
440440- xtrace_memory_fragment_t fragment = __container_of((const char (*)[])old_pointer, struct xtrace_memory_fragment, memory);
441441- size_t old_size = xtrace_memory_fragment_size(fragment);
442442-443443- xtrace_lock_lock(&free_lock);
444444-445445- // if we're shrinking, we can always do that
446446- if (new_size < old_size) {
447447- // we don't really care if it succeeds or not
448448- shrink_fragment(fragment, new_size);
449449- goto out;
450450- } else if (new_size == old_size) {
451451- // likewise, we can always keep it the same
452452- goto out;
453453- }
454454-455455- // but otherwise, we're expanding
456456-457457- // try to see if we can expand the fragment first
458458- if (expand_fragment(fragment, new_size)) {
459459- // if we expanded it successfully, we're done!
460460- goto out;
461461- } else {
462462- // otherwise, we need to allocate a new fragment
463463- xtrace_lock_unlock(&free_lock); // `allocate_fragment` takes the lock, so we need to drop it
464464- xtrace_memory_fragment_t new_fragment = allocate_fragment(new_size);
465465- if (new_fragment == NULL) {
466466- return NULL;
467467- }
468468-469469- // copy the old contents to the new fragment
470470- memcpy(new_fragment->memory, fragment->memory, old_size);
471471-472472- // and release the old fragment
473473- release_fragment(fragment);
474474-475475- // assign the new pointer to the return variable and we're done!
476476- old_pointer = new_fragment->memory;
477477- goto out_unlocked;
478478- }
479479-480480-out:
481481- xtrace_lock_unlock(&free_lock);
482482-483483-out_unlocked:
484484- return old_pointer;
485485-};