···2828#include "caml/memory.h"
2929#include "caml/mlvalues.h"
3030#include "caml/signals.h"
3131-#include "caml/atomic_refcount.h"
3131+#include "caml/camlatomic.h"
32323333#define int8 caml_ba_int8
3434#define uint8 caml_ba_uint8
···292292 if (b->proxy == NULL) {
293293 free(b->data);
294294 } else {
295295- if (caml_atomic_refcount_decr(&b->proxy->refcount) == 1) {
295295+ if (caml_atomic_counter_decr(&b->proxy->refcount) == 0) {
296296 free(b->proxy->data);
297297 free(b->proxy);
298298 }
···10771077 /* If b1 is already a proxy for a larger array, increment refcount of
10781078 proxy */
10791079 b2->proxy = b1->proxy;
10801080- caml_atomic_refcount_incr(&b1->proxy->refcount);
10801080+ (void)caml_atomic_counter_incr(&b1->proxy->refcount);
10811081 } else {
10821082 /* Otherwise, create proxy and attach it to both b1 and b2 */
10831083 proxy = malloc(sizeof(struct caml_ba_proxy));
10841084 if (proxy == NULL) caml_raise_out_of_memory();
10851085- caml_atomic_refcount_init(&proxy->refcount, 2);
10851085+ caml_atomic_counter_init(&proxy->refcount, 2);
10861086 /* initial refcount: 2 = original array + sub array */
10871087 proxy->data = b1->data;
10881088 proxy->size =
-37
runtime/caml/atomic_refcount.h
···11-/**************************************************************************/
22-/* */
33-/* OCaml */
44-/* */
55-/* Florian Angeletti, projet Cambium, Inria */
66-/* */
77-/* Copyright 2022 Institut National de Recherche en Informatique et */
88-/* en Automatique. */
99-/* */
1010-/* All rights reserved. This file is distributed under the terms of */
1111-/* the GNU Lesser General Public License version 2.1, with the */
1212-/* special exception on linking described in the file LICENSE. */
1313-/* */
1414-/**************************************************************************/
1515-1616-#ifndef CAML_ATOMIC_REFCOUNT_H
1717-#define CAML_ATOMIC_REFCOUNT_H
1818-1919-#ifdef CAML_INTERNALS
2020-2121-#include "camlatomic.h"
2222-2323-Caml_inline void caml_atomic_refcount_init(atomic_uintnat* refc, uintnat n){
2424- atomic_store_release(refc, n);
2525-}
2626-2727-Caml_inline uintnat caml_atomic_refcount_decr(atomic_uintnat* refcount){
2828- return atomic_fetch_add (refcount, -1);
2929-}
3030-3131-Caml_inline uintnat caml_atomic_refcount_incr(atomic_uintnat* refcount){
3232- return atomic_fetch_add (refcount, 1);
3333-}
3434-3535-#endif /* CAML_INTERNALS */
3636-3737-#endif // CAML_ATOMIC_REFCOUNT_H
···1717#define CAML_ATOMIC_H
18181919#include "config.h"
2020+#include "misc.h"
20212122/*
2223 * C11 atomics types and utility macros.
···5556 atomic_store_explicit((p), (v), memory_order_release)
5657#define atomic_store_relaxed(p, v) \
5758 atomic_store_explicit((p), (v), memory_order_relaxed)
5959+6060+/* Atomic counters, abstracted here for use across the runtime. */
6161+6262+Caml_inline void caml_atomic_counter_init(atomic_uintnat* counter, uintnat n)
6363+{
6464+ atomic_store_release(counter, n);
6565+}
6666+6767+/* Atomically get the current value of an atomic uintnat counter */
6868+6969+Caml_inline uintnat caml_atomic_counter_value(atomic_uintnat* counter)
7070+{
7171+ return atomic_load_acquire(counter);
7272+}
7373+7474+/* Decrement an atomic uintnat counter. Assertion check for
7575+ * underflow. Returns the new value. */
7676+7777+Caml_inline uintnat caml_atomic_counter_decr(atomic_uintnat* counter)
7878+{
7979+ uintnat old = atomic_fetch_sub(counter, 1);
8080+ CAMLassert (old > 0);
8181+ return old-1;
8282+}
8383+8484+/* Increment an atomic uintnat counter. Assertion check for
8585+ * overflow. Returns the new value. */
8686+8787+Caml_inline uintnat caml_atomic_counter_incr(atomic_uintnat* counter)
8888+{
8989+ uintnat old = atomic_fetch_add(counter, 1);
9090+ CAMLassert (old+1 != 0);
9191+ return old+1;
9292+}
58935994#endif /* CAML_INTERNALS */
6095
+3-3
runtime/caml/misc.h
···2727#include <stdarg.h>
2828#include <limits.h>
29293030-#include "camlatomic.h"
3131-3230/* Detection of available C attributes and compiler extensions */
33313432#ifndef __has_c_attribute
···289287 can obtain the domain id with Caml_state->id. These functions must
290288 be reentrant. */
291289#ifndef __cplusplus
290290+#include <stdatomic.h>
291291+292292typedef void (*caml_timing_hook) (void);
293293extern _Atomic caml_timing_hook caml_major_slice_begin_hook;
294294extern _Atomic caml_timing_hook caml_major_slice_end_hook;
···615615616616/* runtime message flags. Settable with v= in OCAMLRUNPARAM */
617617618618-extern atomic_uintnat caml_verb_gc;
618618+extern _Atomic uintnat caml_verb_gc;
619619620620/* Bits which may be set in caml_verb_gc. The quotations are from the
621621 * OCaml manual. */
···6565}
666667676868-/* Atomic read-modify-write instructions, with full fences */
6969-7070-Caml_inline uintnat atomic_fetch_add_verify_ge0(atomic_uintnat* p, intnat v) {
7171- uintnat result = atomic_fetch_add(p,v);
7272- CAMLassert ((intnat)result > 0);
7373- return result;
7474-}
7575-7668/* Warning: blocking functions.
77697870 Blocking functions are for use in the runtime outside of the
+61-59
runtime/major_gc.c
···296296{
297297 caml_plat_lock_blocking(&ephe_lock);
298298299299- atomic_fetch_add(&ephe_cycle_info.ephe_cycle, +1);
300300- CAMLassert(atomic_load_acquire(&ephe_cycle_info.num_domains_done) <=
301301- atomic_load_acquire(&ephe_cycle_info.num_domains_todo));
302302- atomic_store(&ephe_cycle_info.num_domains_done, 0);
299299+ (void)caml_atomic_counter_incr(&ephe_cycle_info.ephe_cycle);
300300+ CAMLassert(caml_atomic_counter_value(&ephe_cycle_info.num_domains_done) <=
301301+ caml_atomic_counter_value(&ephe_cycle_info.num_domains_todo));
302302+ caml_atomic_counter_init(&ephe_cycle_info.num_domains_done, 0);
303303304304 caml_plat_unlock(&ephe_lock);
305305}
···311311 /* Force next ephemeron marking cycle in order to avoid reasoning about
312312 * whether the domain has already incremented
313313 * [ephe_cycle_info.num_domains_done] counter. */
314314- atomic_store(&ephe_cycle_info.num_domains_done, 0);
315315- atomic_fetch_add(&ephe_cycle_info.ephe_cycle, +1);
314314+ caml_atomic_counter_init(&ephe_cycle_info.num_domains_done, 0);
315315+ (void)caml_atomic_counter_incr(&ephe_cycle_info.ephe_cycle);
316316317317 /* Since the todo list is empty, this domain does not need to participate in
318318 * further ephemeron cycles. */
319319- atomic_fetch_sub(&ephe_cycle_info.num_domains_todo, 1);
320320- CAMLassert(atomic_load_acquire(&ephe_cycle_info.num_domains_done) <=
321321- atomic_load_acquire(&ephe_cycle_info.num_domains_todo));
319319+ (void)caml_atomic_counter_decr(&ephe_cycle_info.num_domains_todo);
320320+ CAMLassert(caml_atomic_counter_value(&ephe_cycle_info.num_domains_done) <=
321321+ caml_atomic_counter_value(&ephe_cycle_info.num_domains_todo));
322322323323 caml_plat_unlock(&ephe_lock);
324324}
···326326/* Record that ephemeron marking was done for the given ephemeron cycle. */
327327static void record_ephe_marking_done (uintnat ephe_cycle)
328328{
329329- CAMLassert (ephe_cycle <= atomic_load_acquire(&ephe_cycle_info.ephe_cycle));
329329+ CAMLassert (ephe_cycle <=
330330+ caml_atomic_counter_value(&ephe_cycle_info.ephe_cycle));
330331 CAMLassert (Caml_state->marking_done);
331332332332- if (ephe_cycle < atomic_load_acquire(&ephe_cycle_info.ephe_cycle))
333333+ if (ephe_cycle < caml_atomic_counter_value(&ephe_cycle_info.ephe_cycle))
333334 return;
334335335336 caml_plat_lock_blocking(&ephe_lock);
336336- if (ephe_cycle == atomic_load(&ephe_cycle_info.ephe_cycle)) {
337337+ if (ephe_cycle == caml_atomic_counter_value(&ephe_cycle_info.ephe_cycle)) {
337338 Caml_state->ephe_info->cycle = ephe_cycle;
338338- atomic_fetch_add(&ephe_cycle_info.num_domains_done, +1);
339339- CAMLassert(atomic_load_acquire(&ephe_cycle_info.num_domains_done) <=
340340- atomic_load_acquire(&ephe_cycle_info.num_domains_todo));
339339+ (void)caml_atomic_counter_incr(&ephe_cycle_info.num_domains_done);
340340+ CAMLassert(caml_atomic_counter_value(&ephe_cycle_info.num_domains_done) <=
341341+ caml_atomic_counter_value(&ephe_cycle_info.num_domains_todo));
341342 }
342343 caml_plat_unlock(&ephe_lock);
343344}
···424425425426 if (ephe_info->must_sweep_ephe) {
426427 ephe_info->must_sweep_ephe = 0;
427427- atomic_fetch_add_verify_ge0(&num_domains_to_ephe_sweep, -1);
428428+ (void)caml_atomic_counter_decr(&num_domains_to_ephe_sweep);
428429 }
429430 CAMLassert (ephe_info->must_sweep_ephe == 0);
430431 CAMLassert (ephe_info->live == 0);
···437438438439 if (f->todo_head != NULL || f->first.size != 0 || f->last.size != 0) {
439440 /* have some final structures */
440440- atomic_fetch_add(&num_domains_orphaning_finalisers, +1);
441441+ (void)caml_atomic_counter_incr(&num_domains_orphaning_finalisers);
441442 if (caml_gc_phase != Phase_sweep_and_mark_main) {
442443 /* Force a major GC cycle to simplify constraints for orphaning
443444 finalisers. See note attached to the declaration of
···456457457458 /* Create a dummy final info */
458459 f = domain_state->final_info = caml_alloc_final_info();
459459- atomic_fetch_add_verify_ge0(&num_domains_orphaning_finalisers, -1);
460460+ (void)caml_atomic_counter_decr(&num_domains_orphaning_finalisers);
460461 }
461462462463 /* [caml_orphan_finalisers] is called in a while loop in
···464465 We take care to decrement the [num_domains_to_final_update*] counters only
465466 if we have not already decremented it for the current cycle. */
466467 if(!f->updated_first) {
467467- atomic_fetch_add_verify_ge0(&num_domains_to_final_update_first, -1);
468468+ (void)caml_atomic_counter_decr(&num_domains_to_final_update_first);
468469 f->updated_first = 1;
469470 }
470471 if(!f->updated_last) {
471471- atomic_fetch_add_verify_ge0(&num_domains_to_final_update_last, -1);
472472+ (void)caml_atomic_counter_decr(&num_domains_to_final_update_last);
472473 f->updated_last = 1;
473474 }
474475}
···11521153 } else {
11531154 ephe_next_cycle ();
11541155 domain_state->marking_done = 1;
11551155- atomic_fetch_add_verify_ge0(&num_domains_to_mark, -1);
11561156+ (void)caml_atomic_counter_decr(&num_domains_to_mark);
11561157 }
11571158 }
11581159 }
···12031204 if (Has_status_hd(hd, caml_global_heap_state.UNMARKED)) {
12041205 caml_domain_state* domain_state = (caml_domain_state*)state;
12051206 if (domain_state->marking_done) {
12061206- atomic_fetch_add(&num_domains_to_mark, 1);
12071207+ (void)caml_atomic_counter_incr(&num_domains_to_mark);
12071208 domain_state->marking_done = 0;
12081209 }
12091210 if (Tag_hd(hd) == Cont_tag) {
···1389139013901391 domain->swept_words = 0;
1391139213921392- atomic_store_release(&num_domains_to_sweep, num_domains_in_stw);
13931393- atomic_store_release(&num_domains_to_mark, num_domains_in_stw);
13931393+ caml_atomic_counter_init(&num_domains_to_sweep, num_domains_in_stw);
13941394+ caml_atomic_counter_init(&num_domains_to_mark, num_domains_in_stw);
1394139513951396 caml_gc_phase = Phase_sweep_and_mark_main;
13961396- atomic_store(&ephe_cycle_info.num_domains_todo, num_domains_in_stw);
13971397- atomic_store(&ephe_cycle_info.ephe_cycle, 1);
13981398- atomic_store(&ephe_cycle_info.num_domains_done, 0);
13971397+ caml_atomic_counter_init(&ephe_cycle_info.num_domains_todo,
13981398+ num_domains_in_stw);
13991399+ caml_atomic_counter_init(&ephe_cycle_info.ephe_cycle, 1);
14001400+ caml_atomic_counter_init(&ephe_cycle_info.num_domains_done, 0);
1399140114001400- atomic_store_release(&num_domains_to_ephe_sweep, 0);
14021402+ caml_atomic_counter_init(&num_domains_to_ephe_sweep, 0);
14011403 /* Will be set to the correct number when switching to
14021404 [Phase_sweep_ephe] */
1403140514041404- atomic_store_release(&num_domains_to_final_update_first,
14051405- num_domains_in_stw);
14061406- atomic_store_release(&num_domains_to_final_update_last,
14071407- num_domains_in_stw);
14061406+ caml_atomic_counter_init(&num_domains_to_final_update_first,
14071407+ num_domains_in_stw);
14081408+ caml_atomic_counter_init(&num_domains_to_final_update_last,
14091409+ num_domains_in_stw);
1408141014091411 atomic_store(&domain_global_roots_started, WORK_UNSTARTED);
14101412···14371439 CAML_EV_BEGIN(EV_MAJOR_GC_CYCLE_DOMAINS);
1438144014391441 CAMLassert(domain == Caml_state);
14401440- CAMLassert(atomic_load_acquire(&ephe_cycle_info.num_domains_todo) ==
14411441- atomic_load_acquire(&ephe_cycle_info.num_domains_done));
14421442- CAMLassert(atomic_load(&num_domains_to_mark) == 0);
14431443- CAMLassert(atomic_load(&num_domains_to_sweep) == 0);
14441444- CAMLassert(atomic_load(&num_domains_to_ephe_sweep) == 0);
14421442+ CAMLassert(caml_atomic_counter_value(&ephe_cycle_info.num_domains_todo) ==
14431443+ caml_atomic_counter_value(&ephe_cycle_info.num_domains_done));
14441444+ CAMLassert(caml_atomic_counter_value(&num_domains_to_mark) == 0);
14451445+ CAMLassert(caml_atomic_counter_value(&num_domains_to_sweep) == 0);
14461446+ CAMLassert(caml_atomic_counter_value(&num_domains_to_ephe_sweep) == 0);
1445144714461448 caml_empty_minor_heap_no_major_slice_from_stw
14471449 (domain, (void*)0, participating_count, participating);
···15171519 !caml_addrmap_iter_ok(&domain->mark_stack->compressed_stack,
15181520 domain->mark_stack->compressed_stack_iter)
15191521 ) {
15201520- atomic_fetch_add_verify_ge0(&num_domains_to_mark, -1);
15221522+ (void)caml_atomic_counter_decr(&num_domains_to_mark);
15211523 domain->marking_done = 1;
15221524 }
15231525···15611563 return
15621564 /* Marking is done */
15631565 caml_gc_phase == Phase_sweep_and_mark_main &&
15641564- atomic_load_acquire (&num_domains_to_sweep) == 0 &&
15651565- atomic_load_acquire (&num_domains_to_mark) == 0 &&
15661566+ caml_atomic_counter_value (&num_domains_to_sweep) == 0 &&
15671567+ caml_atomic_counter_value (&num_domains_to_mark) == 0 &&
1566156815671569 /* No domains are orphaning finalisers. */
15681568- atomic_load_acquire (&num_domains_orphaning_finalisers) == 0 &&
15701570+ caml_atomic_counter_value (&num_domains_orphaning_finalisers) == 0 &&
1569157115701572 /* Ephemeron marking is done */
15711571- atomic_load_acquire(&ephe_cycle_info.num_domains_todo) ==
15721572- atomic_load_acquire(&ephe_cycle_info.num_domains_done) &&
15731573+ caml_atomic_counter_value(&ephe_cycle_info.num_domains_todo) ==
15741574+ caml_atomic_counter_value(&ephe_cycle_info.num_domains_done) &&
1573157515741576 /* All orphaned ephemerons have been adopted */
15751577 no_orphaned_work();
···15801582 return
15811583 /* updated finalise first values */
15821584 caml_gc_phase == Phase_mark_final &&
15831583- atomic_load_acquire (&num_domains_to_final_update_first) == 0 &&
15851585+ caml_atomic_counter_value (&num_domains_to_final_update_first) == 0 &&
1584158615851587 /* Marking is done */
15861586- atomic_load_acquire (&num_domains_to_mark) == 0 &&
15881588+ caml_atomic_counter_value (&num_domains_to_mark) == 0 &&
1587158915881590 /* Ephemeron marking is done */
15891589- atomic_load_acquire(&ephe_cycle_info.num_domains_todo) ==
15901590- atomic_load_acquire(&ephe_cycle_info.num_domains_done) &&
15911591+ caml_atomic_counter_value(&ephe_cycle_info.num_domains_todo) ==
15921592+ caml_atomic_counter_value(&ephe_cycle_info.num_domains_done) &&
1591159315921594 /* All orphaned ephemerons have been adopted */
15931595 no_orphaned_work();
···15981600 return
15991601 /* All domains have swept their ephemerons */
16001602 caml_gc_phase == Phase_sweep_ephe &&
16011601- atomic_load_acquire (&num_domains_to_ephe_sweep) == 0 &&
16031603+ caml_atomic_counter_value (&num_domains_to_ephe_sweep) == 0 &&
1602160416031605 /* All domains have updated finalise last values */
16041604- atomic_load_acquire (&num_domains_to_final_update_last) == 0 &&
16061606+ caml_atomic_counter_value (&num_domains_to_final_update_last) == 0 &&
1605160716061608 /* All orphaned structures have been adopted */
16071609 no_orphaned_work();
···16191621 caml_gc_phase = Phase_mark_final;
16201622 } else if (is_complete_phase_mark_final()) {
16211623 caml_gc_phase = Phase_sweep_ephe;
16221622- atomic_store_release(&num_domains_to_ephe_sweep, participant_count);
16241624+ caml_atomic_counter_init(&num_domains_to_ephe_sweep, participant_count);
16231625 for (int i = 0; i < participant_count; i++)
16241626 participating[i]->ephe_info->must_sweep_ephe = 1;
16251627 }
···16961698 commit_major_slice_work (work_done);
16971699 if (work_done == 0) {
16981700 domain_state->sweeping_done = 1;
16991699- atomic_fetch_add_verify_ge0(&num_domains_to_sweep, -1);
17011701+ (void)caml_atomic_counter_decr(&num_domains_to_sweep);
17001702 }
17011703 }
17021704···17251727 get_major_slice_work(mode) > 0 &&
17261728 caml_final_update_first(domain_state)) {
17271729 /* This domain has updated finalise first values */
17281728- atomic_fetch_add_verify_ge0(&num_domains_to_final_update_first, -1);
17301730+ (void)caml_atomic_counter_decr(&num_domains_to_final_update_first);
17291731 if (!domain_state->marking_done &&
17301732 get_major_slice_work(mode) > 0)
17311733 goto mark_again;
···17351737 get_major_slice_work(mode) > 0 &&
17361738 caml_final_update_last(domain_state)) {
17371739 /* This domain has updated finalise last values */
17381738- atomic_fetch_add_verify_ge0(&num_domains_to_final_update_last, -1);
17401740+ (void)caml_atomic_counter_decr(&num_domains_to_final_update_last);
17391741 /* Nothing has been marked while updating last */
17401742 }
17411743···17471749 /* Ephemerons */
17481750 if (caml_gc_phase != Phase_sweep_ephe) {
17491751 /* Ephemeron Marking */
17501750- saved_ephe_cycle = atomic_load_acquire(&ephe_cycle_info.ephe_cycle);
17521752+ saved_ephe_cycle = caml_atomic_counter_value(&ephe_cycle_info.ephe_cycle);
17511753 if (domain_state->ephe_info->todo != (value) NULL &&
17521754 saved_ephe_cycle > domain_state->ephe_info->cycle &&
17531755 get_major_slice_work(mode) > 0) {
···18051807 /* If the todo list is empty, then the ephemeron has no sweeping work
18061808 * to do. */
18071809 if (domain_state->ephe_info->todo == 0) {
18081808- atomic_fetch_add_verify_ge0(&num_domains_to_ephe_sweep, -1);
18101810+ (void)caml_atomic_counter_decr(&num_domains_to_ephe_sweep);
18091811 }
18101812 }
18111813···1823182518241826 CAML_EV_END(EV_MAJOR_EPHE_SWEEP);
18251827 if (domain_state->ephe_info->todo == 0) {
18261826- atomic_fetch_add_verify_ge0(&num_domains_to_ephe_sweep, -1);
18281828+ (void)caml_atomic_counter_decr(&num_domains_to_ephe_sweep);
18271829 }
18281830 }
18291831 }
···20002002 /* just finished sweeping */
20012003 CAMLassert(Caml_state->sweeping_done == 0);
20022004 Caml_state->sweeping_done = 1;
20032003- atomic_fetch_add_verify_ge0(&num_domains_to_sweep, -1);
20052005+ (void)caml_atomic_counter_decr(&num_domains_to_sweep);
20042006 break;
20052007 }
20062008 caml_handle_incoming_interrupts();
···21202122 d->mark_stack = NULL;
21212123 return -1;
21222124 }
21232123- atomic_fetch_add(&num_domains_to_final_update_first, 1);
21242124- atomic_fetch_add(&num_domains_to_final_update_last, 1);
21252125+ (void)caml_atomic_counter_incr(&num_domains_to_final_update_first);
21262126+ (void)caml_atomic_counter_incr(&num_domains_to_final_update_last);
2125212721262128 return 0;
21272129}