this repo has no description
Improve xtrace output for posix_spawn and fix(?) xtrace TLS
This makes xtrace print more useful debugging information for posix_spawn. There are other syscalls that could definitely benefit from having more detailed output (e.g. `kevent` and friends).
Also, I was noticing segfaults when processes exited with pthread_terminate; they were due to xtrace using pthread keys after pthread_terminate had been called, which cleans up pthread keys. This new static key approach should be fault free, if only at the expense of leaking some memory per-thread. TODO: add a death hook in libsystem_kernel to notify xtrace when a thread is *actually* going to die.
+175
-17
+1
src/xtrace/CMakeLists.txt
+1
src/xtrace/CMakeLists.txt
+36
-2
src/xtrace/bsd_trace.cpp
+36
-2
src/xtrace/bsd_trace.cpp
···
4
4
#include <sys/mman.h>
5
5
#include <pthread.h>
6
6
7
+
#define PRIVATE 1
8
+
#include <spawn_private.h>
9
+
7
10
#include "xtracelib.h"
8
11
#include "bsd_trace.h"
9
12
···
430
433
return __simple_sprintf(buf, "%p", arg);
431
434
}
432
435
436
+
extern "C"
433
437
int xtrace_format_string_literal(char* buf, const char* str)
434
438
{
435
439
const char* initial_buf = buf;
···
551
555
return buf - initial_buf;
552
556
}
553
557
554
-
static int print_open_flags(char* buf, void* arg)
558
+
extern "C"
559
+
int print_open_flags(char* buf, void* arg)
555
560
{
556
561
const char* initial_buf = buf;
557
562
int cnt = 0;
···
598
603
return buf - initial_buf;
599
604
}
600
605
606
+
extern "C" int print_arg_posix_spawn_args(char* buf, void* arg);
607
+
608
+
static int print_arg_string_array(char* buf, void* arg) {
609
+
const char* initial_buf = buf;
610
+
const char* const* argv = (const char* const*)arg;
611
+
bool is_first = true;
612
+
613
+
*buf++ = '{';
614
+
615
+
for (const char* const* ptr = argv; *ptr != NULL; ++ptr) {
616
+
if (is_first) {
617
+
is_first = false;
618
+
} else {
619
+
*buf++ = ',';
620
+
*buf++ = ' ';
621
+
}
622
+
623
+
buf += xtrace_format_string_literal(buf, *ptr);
624
+
}
625
+
626
+
*buf++ = '}';
627
+
628
+
return buf - initial_buf;
629
+
};
630
+
631
+
// TODO: output more specific information for certain calls
632
+
// e.g. the kqueue family of syscalls would definitely be great candidates for this.
633
+
// that way, we can see what events have been returned and what the application might be processing now.
634
+
601
635
static const struct {
602
636
int args_cnt;
603
637
int (*print_arg[8])(char* buf, void* arg);
···
765
799
[241] = { 4, { print_arg_int, print_arg_ptr, print_arg_int, print_arg_int } }, // flistxattr
766
800
[242] = { 4, { print_arg_str, print_arg_int, print_arg_ptr, print_arg_int } }, // fsctl
767
801
[243] = { 3, { print_arg_int, print_arg_ptr, print_arg_int } }, // initgroups
768
-
[244] = { 5, { print_arg_ptr, print_arg_str, print_arg_ptr, print_arg_ptr, print_arg_ptr } }, // posix_spawn
802
+
[244] = { 5, { print_arg_ptr, print_arg_str, print_arg_posix_spawn_args, print_arg_string_array, print_arg_string_array } }, // posix_spawn
769
803
[245] = { 4, { print_arg_int, print_arg_int, print_arg_ptr, print_arg_int } }, // ffsctl
770
804
[247] = { 2, { print_arg_int, print_arg_ptr } }, // nfsclnt
771
805
[248] = { 2, { print_arg_ptr, print_arg_int } }, // fhopen
+120
src/xtrace/posix_spawn_args.c
+120
src/xtrace/posix_spawn_args.c
···
1
+
/*
2
+
* This function needs to be in a separate C file rather than in bsd_trace.cpp
3
+
* because `sys/spawn_internal.h` uses some GNU C extensions that C++ doesn't seem to like.
4
+
*/
5
+
6
+
#define PRIVATE 1
7
+
#include <sys/spawn_internal.h>
8
+
#include <spawn_private.h>
9
+
10
+
#include <darling/emulation/simple.h>
11
+
12
+
extern int xtrace_format_string_literal(char* buf, const char* str);
13
+
extern int print_open_flags(char* buf, void* arg);
14
+
15
+
static struct {
16
+
unsigned short flag;
17
+
const char* name;
18
+
} all_posix_spawn_attrs[] = {
19
+
#define POSIX_SPAWN_ATTR_ENTRY(name) { name, #name }
20
+
POSIX_SPAWN_ATTR_ENTRY(POSIX_SPAWN_RESETIDS),
21
+
POSIX_SPAWN_ATTR_ENTRY(POSIX_SPAWN_SETPGROUP),
22
+
POSIX_SPAWN_ATTR_ENTRY(POSIX_SPAWN_SETSIGDEF),
23
+
POSIX_SPAWN_ATTR_ENTRY(POSIX_SPAWN_SETSIGMASK),
24
+
POSIX_SPAWN_ATTR_ENTRY(POSIX_SPAWN_SETEXEC),
25
+
POSIX_SPAWN_ATTR_ENTRY(POSIX_SPAWN_START_SUSPENDED),
26
+
POSIX_SPAWN_ATTR_ENTRY(_POSIX_SPAWN_DISABLE_ASLR),
27
+
POSIX_SPAWN_ATTR_ENTRY(_POSIX_SPAWN_NANO_ALLOCATOR),
28
+
POSIX_SPAWN_ATTR_ENTRY(POSIX_SPAWN_SETSID),
29
+
POSIX_SPAWN_ATTR_ENTRY(_POSIX_SPAWN_ALLOW_DATA_EXEC),
30
+
POSIX_SPAWN_ATTR_ENTRY(POSIX_SPAWN_CLOEXEC_DEFAULT),
31
+
POSIX_SPAWN_ATTR_ENTRY(_POSIX_SPAWN_HIGH_BITS_ASLR),
32
+
{ 0, NULL },
33
+
#undef POSIX_SPAWN_ATTR_ENTRY
34
+
};
35
+
36
+
int print_arg_posix_spawn_args(char* buf, void* arg) {
37
+
const char* initial_buf = buf;
38
+
const struct _posix_spawn_args_desc* args = (const struct _posix_spawn_args_desc*)(arg);
39
+
bool is_first = true;
40
+
41
+
buf += __simple_sprintf(buf, "{ attributes = ");
42
+
43
+
if (args && args->attrp) {
44
+
for (size_t i = 0; all_posix_spawn_attrs[i].name != NULL; i++) {
45
+
if (args->attrp->psa_flags & all_posix_spawn_attrs[i].flag) {
46
+
if (is_first) {
47
+
is_first = false;
48
+
} else {
49
+
*buf++ = '|';
50
+
}
51
+
52
+
buf += __simple_sprintf(buf, "%s", all_posix_spawn_attrs[i].name);
53
+
}
54
+
}
55
+
}
56
+
57
+
if (is_first) {
58
+
*buf++ = '0';
59
+
}
60
+
61
+
buf += __simple_sprintf(buf, ", file_actions = {");
62
+
63
+
if (args && args->file_actions) {
64
+
for (size_t i = 0; i < args->file_actions->psfa_act_count; ++i) {
65
+
const struct _psfa_action* action = &args->file_actions->psfa_act_acts[i];
66
+
67
+
if (i != 0) {
68
+
*buf++ = ',';
69
+
*buf++ = ' ';
70
+
}
71
+
72
+
switch (action->psfaa_type) {
73
+
case PSFA_OPEN:
74
+
buf += __simple_sprintf(buf, "open(");
75
+
buf += xtrace_format_string_literal(buf, action->psfaa_openargs.psfao_path);
76
+
*buf++ = ',';
77
+
*buf++ = ' ';
78
+
buf += print_open_flags(buf, (void*)(intptr_t)(action->psfaa_openargs.psfao_oflag));
79
+
buf += __simple_sprintf(buf, ", 0%o) to %d", action->psfaa_openargs.psfao_mode, action->psfaa_filedes);
80
+
break;
81
+
82
+
case PSFA_CLOSE:
83
+
buf += __simple_sprintf(buf, "close(%d)", action->psfaa_filedes);
84
+
break;
85
+
86
+
case PSFA_DUP2:
87
+
buf += __simple_sprintf(buf, "dup2(%d, %d)", action->psfaa_filedes, action->psfaa_dup2args.psfad_newfiledes);
88
+
break;
89
+
90
+
case PSFA_INHERIT:
91
+
buf += __simple_sprintf(buf, "inherit(%d)", action->psfaa_filedes);
92
+
break;
93
+
94
+
case PSFA_FILEPORT_DUP2:
95
+
// NOTE: if we see this in the output, that's an issue;
96
+
// we don't have this implemented right now
97
+
buf += __simple_sprintf(buf, "dup2_fileport(port right %d, %d)", action->psfaa_fileport, action->psfaa_dup2args.psfad_newfiledes);
98
+
break;
99
+
100
+
case PSFA_CHDIR:
101
+
buf += __simple_sprintf(buf, "chdir(");
102
+
buf += xtrace_format_string_literal(buf, action->psfaa_chdirargs.psfac_path);
103
+
*buf++ = ')';
104
+
break;
105
+
106
+
case PSFA_FCHDIR:
107
+
buf += __simple_sprintf(buf, "fchdir(%d)", action->psfaa_filedes);
108
+
break;
109
+
110
+
default:
111
+
buf += __simple_sprintf(buf, "???");
112
+
break;
113
+
}
114
+
}
115
+
}
116
+
117
+
buf += __simple_sprintf(buf, "} }");
118
+
119
+
return buf - initial_buf;
120
+
};
+18
-15
src/xtrace/tls.c
+18
-15
src/xtrace/tls.c
···
1
-
#include <pthread/pthread.h>
2
1
#include <stdlib.h>
3
2
#include <darling/emulation/ext/for-xtrace.h>
4
3
#include "tls.h"
5
4
#include "malloc.h"
6
5
#include "lock.h"
7
6
#include <darling/emulation/simple.h>
7
+
#include <pthread/tsd_private.h>
8
8
9
9
#ifndef XTRACE_TLS_DEBUG
10
10
#define XTRACE_TLS_DEBUG 0
···
16
16
#define xtrace_tls_debug(x, ...)
17
17
#endif
18
18
19
-
// 100 TLS vars should be enough, right?
20
-
#define TLS_TABLE_MAX_SIZE 100
19
+
// 10 TLS vars should be enough, right?
20
+
#define TLS_TABLE_MAX_SIZE 10
21
21
22
22
typedef struct tls_table* tls_table_t;
23
23
struct tls_table {
···
25
25
void* table[TLS_TABLE_MAX_SIZE][2];
26
26
};
27
27
28
-
static xtrace_once_t tls_key_initialized = XTRACE_ONCE_INITIALIZER;
29
-
static pthread_key_t tls_key;
28
+
// since we still need to handle some calls after pthread_terminate is called and libpthread unwinds its TLS right before calling pthread_terminate,
29
+
// we have to use a slightly hackier technique: using one of the system's reserved but unused TLS keys.
30
+
// key 200 seems like a good fit; it's in the reserved region but it's not currently listed in `pthread/tsd_private.h` as being in-use by anything.
31
+
32
+
#define __PTK_XTRACE_TLS 200
33
+
34
+
// unfortunately, this approach also means that we can't automatically free the TLS memory when the thread dies, since the TLS table needs to stay alive after pthread_terminate.
35
+
// in order to clean up the memory without a pthread key destructor, we'd need to modify our libsystem_kernel to inform us (via a hook) in every case where the thread could die.
36
+
//
37
+
// leaking a bit of memory per-thread being xtrace'd shouldn't be a big problem, unless the tracee is creating and terminating threads very quickly.
38
+
// but it'd be nice if this could eventually be fixed (probably by adding a death hook to libsystem_kernel, as described above).
30
39
40
+
#if 0
31
41
static void tls_table_destructor(void* _table) {
32
42
tls_table_t table = _table;
33
43
xtrace_tls_debug("destroying table %p", table);
···
38
48
xtrace_tls_debug("freeing table %p", table);
39
49
xtrace_free(table);
40
50
};
41
-
42
-
static void tls_key_initializer(void) {
43
-
xtrace_tls_debug("initializing tls key");
44
-
if (pthread_key_create(&tls_key, tls_table_destructor) != 0) {
45
-
_abort_with_payload_for_xtrace(0, 0, NULL, 0, "xtrace: failed pthread key creation", 0);
46
-
}
47
-
};
51
+
#endif
48
52
49
53
void* xtrace_tls(void* key, size_t size) {
50
54
xtrace_tls_debug("looking up tls variable for key %p", key);
51
55
52
-
xtrace_once(&tls_key_initialized, tls_key_initializer);
53
-
tls_table_t table = pthread_getspecific(tls_key);
56
+
tls_table_t table = _pthread_getspecific_direct(__PTK_XTRACE_TLS);
54
57
55
58
xtrace_tls_debug("got %p as table pointer from pthread", table);
56
59
···
62
65
_abort_with_payload_for_xtrace(0, 0, NULL, 0, "xtrace: failed TLS table memory allocation", 0);
63
66
}
64
67
table->size = 0;
65
-
pthread_setspecific(tls_key, table);
68
+
_pthread_setspecific_direct(__PTK_XTRACE_TLS, table);
66
69
}
67
70
68
71
// check if the key is already present