The unpac monorepo manager self-hosting as a monorepo using unpac
Refactor Dynlink startup to avoid parsing bytecode sections twice (#12599)
This removes the dependency from Symtable->Bytesections, because now
Dynlink and toplevel startup can ask the runtime for the bytecode
sections that were parsed at startup time, rather than re-parsing
them in OCaml.
authored by
Stephen Dolan
and committed by
GitHub
b851fea9
1680bfe2
+132
-105
-2
.depend
-2
.depend
···
2259
2259
utils/config.cmi \
2260
2260
file_formats/cmo_format.cmi \
2261
2261
utils/clflags.cmi \
2262
-
bytecomp/bytesections.cmi \
2263
2262
bytecomp/symtable.cmi
2264
2263
bytecomp/symtable.cmx : \
2265
2264
lambda/runtimedef.cmx \
···
2273
2272
utils/config.cmx \
2274
2273
file_formats/cmo_format.cmi \
2275
2274
utils/clflags.cmx \
2276
-
bytecomp/bytesections.cmx \
2277
2275
bytecomp/symtable.cmi
2278
2276
bytecomp/symtable.cmi : \
2279
2277
lambda/lambda.cmi \
+4
Changes
+4
Changes
···
245
245
(KC Sivaramakrishnan, report by Gabriel Scherer, review by Gabriel Scherer,
246
246
Sadiq Jaffer and Fabrice Buoro)
247
247
248
+
- #12599: Refactor Dynlink startup to avoid parsing bytecode sections twice
249
+
(Stephen Dolan, review by David Allsopp, Hugo Heuzard, Damien Doligez and
250
+
Xavier Leroy)
251
+
248
252
- #12681: Fix TSan false positives due to volatile write handling
249
253
(Olivier Nicole, Fabrice Buoro and Anmol Sahoo, review by Luc Maranget,
250
254
Gabriel Scherer, Hernan Ponce de Leon and Xavier Leroy)
+5
-7
bytecomp/bytelink.ml
+5
-7
bytecomp/bytelink.ml
···
496
496
};
497
497
|};
498
498
(* The sections *)
499
-
let sections : (string * Obj.t) list =
500
-
[ Bytesections.Name.to_string SYMB,
501
-
Symtable.data_global_map();
502
-
Bytesections.Name.to_string PRIM,
503
-
Obj.repr(Symtable.data_primitive_names());
504
-
Bytesections.Name.to_string CRCS,
505
-
Obj.repr(extract_crc_interfaces()) ]
499
+
let sections : (string * Obj.t) array =
500
+
[| Bytesections.Name.to_string SYMB,
501
+
Symtable.data_global_map();
502
+
Bytesections.Name.to_string CRCS,
503
+
Obj.repr(extract_crc_interfaces()) |]
506
504
in
507
505
output_string outchan {|
508
506
static char caml_sections[] = {
+1
-4
bytecomp/dll.ml
+1
-4
bytecomp/dll.ml
···
174
174
(* Initialization for linking in core (dynlink or toplevel) *)
175
175
176
176
let init_toplevel dllpaths =
177
-
search_path :=
178
-
ld_library_path_contents() @
179
-
dllpaths @
180
-
ld_conf_contents();
177
+
search_path := dllpaths;
181
178
opened_dlls :=
182
179
List.map (fun dll -> "", Execution dll)
183
180
(Array.to_list (get_current_dlls()));
-2
bytecomp/meta.ml
-2
bytecomp/meta.ml
-2
bytecomp/meta.mli
-2
bytecomp/meta.mli
+14
-56
bytecomp/symtable.ml
+14
-56
bytecomp/symtable.ml
···
164
164
let data_primitive_names () =
165
165
all_primitives()
166
166
|> Array.to_list
167
-
|> concat_null_terminated
168
167
169
168
let output_primitive_names outchan =
170
-
output_string outchan (data_primitive_names())
169
+
output_string outchan (data_primitive_names() |> concat_null_terminated)
171
170
172
171
open Printf
173
172
···
336
335
!literal_table;
337
336
literal_table := []
338
337
339
-
(* Recover data for toplevel initialization. Data can come either from
340
-
executable file (normal case) or from linked-in data (-output-obj). *)
338
+
type bytecode_sections =
339
+
{ symb: GlobalMap.t;
340
+
crcs: (string * Digest.t option) list;
341
+
prim: string list;
342
+
dlpt: string list }
341
343
342
-
type section_reader = {
343
-
read_string: Bytesections.Name.t -> string;
344
-
read_struct: Bytesections.Name.t -> Obj.t;
345
-
close_reader: unit -> unit
346
-
}
347
-
348
-
let read_sections () =
349
-
try
350
-
let sections =
351
-
List.map
352
-
(fun (n,o) -> Bytesections.Name.of_string n, o)
353
-
(Meta.get_section_table ())
354
-
in
355
-
{ read_string =
356
-
(fun name ->
357
-
(Obj.magic(List.assoc name sections) : string));
358
-
read_struct =
359
-
(fun name -> List.assoc name sections);
360
-
close_reader =
361
-
(fun () -> ()) }
362
-
with Not_found ->
363
-
let ic = open_in_bin Sys.executable_name in
364
-
let section_table = Bytesections.read_toc ic in
365
-
{ read_string = Bytesections.read_section_string section_table ic;
366
-
read_struct = Bytesections.read_section_struct section_table ic;
367
-
close_reader = fun () -> close_in ic }
344
+
external get_bytecode_sections : unit -> bytecode_sections =
345
+
"caml_dynlink_get_bytecode_sections"
368
346
369
347
(* Initialize the linker for toplevel use *)
370
348
371
349
let init_toplevel () =
372
-
try
373
-
let sect = read_sections () in
374
-
(* Locations of globals *)
375
-
global_table :=
376
-
(Obj.magic (sect.read_struct Bytesections.Name.SYMB) : GlobalMap.t);
377
-
(* Primitives *)
378
-
let prims =
379
-
Misc.split_null_terminated (sect.read_string Bytesections.Name.PRIM) in
380
-
c_prim_table := PrimMap.empty;
381
-
List.iter set_prim_table prims;
382
-
(* DLL initialization *)
383
-
let dllpaths =
384
-
try Misc.split_null_terminated (sect.read_string Bytesections.Name.DLPT)
385
-
with Not_found -> [] in
386
-
Dll.init_toplevel dllpaths;
387
-
(* Recover CRC infos for interfaces *)
388
-
let crcintfs =
389
-
try
390
-
(Obj.magic (sect.read_struct Bytesections.Name.CRCS)
391
-
: (string * Digest.t option) list)
392
-
with Not_found -> [] in
393
-
(* Done *)
394
-
sect.close_reader();
395
-
crcintfs
396
-
with Bytesections.Bad_magic_number | Not_found | Failure _ ->
397
-
fatal_error "Toplevel bytecode executable is corrupted"
350
+
let sect = get_bytecode_sections () in
351
+
global_table := sect.symb;
352
+
c_prim_table := PrimMap.empty;
353
+
List.iter set_prim_table sect.prim;
354
+
Dll.init_toplevel sect.dlpt;
355
+
sect.crcs
398
356
399
357
(* Find the value of a global identifier *)
400
358
+1
-1
bytecomp/symtable.mli
+1
-1
bytecomp/symtable.mli
···
55
55
val output_primitive_names: out_channel -> unit
56
56
val output_primitive_table: out_channel -> unit
57
57
val data_global_map: unit -> Obj.t
58
-
val data_primitive_names: unit -> string
58
+
val data_primitive_names: unit -> string list
59
59
val transl_const: Lambda.structured_constant -> Obj.t
60
60
61
61
(* Functions for the toplevel *)
+1
runtime/caml/startup.h
+1
runtime/caml/startup.h
···
41
41
42
42
extern int caml_attempt_open(char_os **name, struct exec_trailer *trail,
43
43
int do_open_script);
44
+
extern int caml_read_trailer(int fd, struct exec_trailer *trail);
44
45
extern void caml_read_section_descriptors(int fd, struct exec_trailer *trail);
45
46
extern int32_t caml_seek_optional_section(int fd, struct exec_trailer *trail,
46
47
char *name);
+103
-29
runtime/dynlink.c
+103
-29
runtime/dynlink.c
···
35
35
#include "caml/osdeps.h"
36
36
#include "caml/prims.h"
37
37
#include "caml/signals.h"
38
+
#include "caml/intext.h"
39
+
#include "caml/startup.h"
38
40
39
41
#include "build_config.h"
40
42
41
43
#ifndef NATIVE_CODE
42
44
45
+
#ifndef O_BINARY
46
+
#define O_BINARY 0
47
+
#endif
48
+
43
49
/* The table of primitives */
44
50
struct ext_table caml_prim_table;
45
51
46
-
#ifdef DEBUG
47
-
/* The names of primitives (for instrtrace.c) */
52
+
/* The names of primitives */
48
53
struct ext_table caml_prim_name_table;
49
-
#endif
50
54
51
55
/* The table of shared libraries currently opened */
52
56
static struct ext_table shared_libs;
···
164
168
char_os * libs,
165
169
char * req_prims)
166
170
{
167
-
char_os * tofree1, * tofree2;
168
171
char_os * p;
169
172
char * q;
170
173
···
172
175
- directories specified on the command line with the -I option
173
176
- directories specified in the CAML_LD_LIBRARY_PATH
174
177
- directories specified in the executable
175
-
- directories specified in the file <stdlib>/ld.conf */
176
-
tofree1 = caml_decompose_path(&caml_shared_libs_path,
177
-
caml_secure_getenv(T("CAML_LD_LIBRARY_PATH")));
178
+
- directories specified in the file <stdlib>/ld.conf
179
+
180
+
caml_shared_libs_path and caml_prim_name_table are not freed afterwards:
181
+
they may later be used by caml_dynlink_get_bytecode_sections. */
182
+
caml_decompose_path(&caml_shared_libs_path,
183
+
caml_secure_getenv(T("CAML_LD_LIBRARY_PATH")));
178
184
if (lib_path != NULL)
179
185
for (p = lib_path; *p != 0; p += strlen_os(p) + 1)
180
186
caml_ext_table_add(&caml_shared_libs_path, p);
181
-
tofree2 = caml_parse_ld_conf();
187
+
caml_parse_ld_conf();
182
188
/* Open the shared libraries */
183
189
caml_ext_table_init(&shared_libs, 8);
184
190
if (libs != NULL)
···
186
192
open_shared_lib(p);
187
193
/* Build the primitive table */
188
194
caml_ext_table_init(&caml_prim_table, 0x180);
189
-
#ifdef DEBUG
190
195
caml_ext_table_init(&caml_prim_name_table, 0x180);
191
-
#endif
192
-
for (q = req_prims; *q != 0; q += strlen(q) + 1) {
193
-
c_primitive prim = lookup_primitive(q);
194
-
if (prim == NULL)
195
-
caml_fatal_error("unknown C primitive `%s'", q);
196
-
caml_ext_table_add(&caml_prim_table, (void *) prim);
197
-
#ifdef DEBUG
198
-
caml_ext_table_add(&caml_prim_name_table, caml_stat_strdup(q));
199
-
#endif
200
-
}
201
-
/* Clean up */
202
-
caml_stat_free(tofree1);
203
-
caml_stat_free(tofree2);
204
-
caml_ext_table_free(&caml_shared_libs_path, 0);
196
+
if (req_prims != NULL)
197
+
for (q = req_prims; *q != 0; q += strlen(q) + 1) {
198
+
c_primitive prim = lookup_primitive(q);
199
+
if (prim == NULL)
200
+
caml_fatal_error("unknown C primitive `%s'", q);
201
+
caml_ext_table_add(&caml_prim_table, (void *) prim);
202
+
caml_ext_table_add(&caml_prim_name_table, caml_stat_strdup(q));
203
+
}
205
204
}
206
205
207
206
/* Build the table of primitives as a copy of the builtin primitive table.
···
210
209
void caml_build_primitive_table_builtin(void)
211
210
{
212
211
int i;
213
-
caml_ext_table_init(&caml_prim_table, 0x180);
214
-
#ifdef DEBUG
215
-
caml_ext_table_init(&caml_prim_name_table, 0x180);
216
-
#endif
212
+
caml_build_primitive_table(NULL, NULL, NULL);
217
213
for (i = 0; caml_builtin_cprim[i] != 0; i++) {
218
214
caml_ext_table_add(&caml_prim_table, (void *) caml_builtin_cprim[i]);
219
-
#ifdef DEBUG
220
215
caml_ext_table_add(&caml_prim_name_table,
221
216
caml_stat_strdup(caml_names_of_builtin_cprim[i]));
222
-
#endif
223
217
}
224
218
}
225
219
···
229
223
caml_dlclose(shared_libs.contents[--shared_libs.size]);
230
224
}
231
225
226
+
CAMLprim value caml_dynlink_get_bytecode_sections(value unit)
227
+
{
228
+
CAMLparam1(unit);
229
+
CAMLlocal4(ret, tbl, list, str);
230
+
int i, j;
231
+
ret = caml_alloc(4, 0);
232
+
233
+
if (caml_params->section_table != NULL) {
234
+
/* cf. Symtable.bytecode_sections */
235
+
const char* sec_names[] = {"SYMB", "CRCS"};
236
+
tbl = caml_input_value_from_block(caml_params->section_table,
237
+
caml_params->section_table_size);
238
+
for (i = 0; i < sizeof(sec_names)/sizeof(sec_names[0]); i++) {
239
+
for (j = 0; j < Wosize_val(tbl); j++) {
240
+
value kv = Field(tbl, j);
241
+
if (!strcmp(sec_names[i], String_val(Field(kv, 0))))
242
+
Store_field(ret, i, Field(kv, 1));
243
+
}
244
+
}
245
+
} else {
246
+
struct exec_trailer trail;
247
+
int fd, err;
248
+
char *sect;
249
+
int32_t len;
250
+
251
+
fd = open_os(caml_params->exe_name, O_RDONLY | O_BINARY);
252
+
if (fd < 0)
253
+
caml_failwith("Dynlink: Failed to re-open bytecode executable");
254
+
255
+
err = caml_read_trailer(fd, &trail);
256
+
if (err != 0)
257
+
caml_failwith("Dynlink: Failed to re-read bytecode trailer");
258
+
259
+
caml_read_section_descriptors(fd, &trail);
260
+
261
+
len = caml_seek_optional_section(fd, &trail, "SYMB");
262
+
sect = caml_stat_alloc(len);
263
+
if (read(fd, sect, len) != len)
264
+
caml_failwith("Dynlink: error reading SYMB");
265
+
Store_field(ret, 0,
266
+
caml_input_value_from_block(sect, len));
267
+
caml_stat_free(sect);
268
+
269
+
len = caml_seek_optional_section(fd, &trail, "CRCS");
270
+
if (len > 0) {
271
+
sect = caml_stat_alloc(len);
272
+
if (read(fd, sect, len) != len)
273
+
caml_failwith("Dynlink: error reading CRCS");
274
+
Store_field(ret, 1,
275
+
caml_input_value_from_block(sect, len));
276
+
caml_stat_free(sect);
277
+
}
278
+
279
+
caml_stat_free(trail.section);
280
+
close(fd);
281
+
}
282
+
283
+
list = Val_emptylist;
284
+
for (i = caml_prim_name_table.size - 1; i >= 0; i--) {
285
+
str = caml_copy_string(caml_prim_name_table.contents[i]);
286
+
list = caml_alloc_2(Tag_cons, str, list);
287
+
}
288
+
Store_field(ret, 2, list);
289
+
290
+
list = Val_emptylist;
291
+
for (i = caml_shared_libs_path.size - 1; i >= 0; i--) {
292
+
str = caml_copy_string_of_os(caml_shared_libs_path.contents[i]);
293
+
list = caml_alloc_2(Tag_cons, str, list);
294
+
}
295
+
Store_field(ret, 3, list);
296
+
297
+
CAMLreturn (ret);
298
+
}
299
+
232
300
#endif /* NATIVE_CODE */
233
301
234
302
/** dlopen interface for the bytecode linker **/
···
307
375
value caml_dynlink_get_current_libs(value unit)
308
376
{
309
377
caml_invalid_argument("dynlink_get_current_libs");
378
+
return Val_unit; /* not reached */
379
+
}
380
+
381
+
value caml_dynlink_get_bytecode_sections(value unit)
382
+
{
383
+
caml_invalid_argument("dynlink_get_bytecode_sections");
310
384
return Val_unit; /* not reached */
311
385
}
312
386
+3
-2
runtime/startup_byt.c
+3
-2
runtime/startup_byt.c
···
93
93
#endif
94
94
}
95
95
96
-
static int read_trailer(int fd, struct exec_trailer *trail)
96
+
int caml_read_trailer(int fd, struct exec_trailer *trail)
97
97
{
98
98
if (lseek(fd, (long) -TRAILER_SIZE, SEEK_END) == -1)
99
99
return BAD_BYTECODE;
···
144
144
return BAD_BYTECODE;
145
145
}
146
146
}
147
-
err = read_trailer(fd, trail);
147
+
err = caml_read_trailer(fd, trail);
148
148
if (err != 0) {
149
149
close(fd);
150
150
caml_stat_free(truename);
···
624
624
#endif
625
625
caml_init_custom_operations();
626
626
caml_init_os_params();
627
+
caml_ext_table_init(&caml_shared_libs_path, 8);
627
628
628
629
/* Initialize the abstract machine */
629
630
caml_init_gc ();