this repo has no description
Acquire semaphore outside of goroutine for resyncs (#668)
Currently we end up with millions of goroutines waiting on semaphores
during a full PDS resync if we run against all production PDSs, this fix
waits for the semaphore _before_ spawning the goroutine reducing allocs
and the size of the stack significantly.
+5
-6
+5
-6
bgs/bgs.go
+5
-6
bgs/bgs.go
···
1449
1449
1450
1450
// Check repo revs against our local copy and enqueue crawls for any that are out of date
1451
1451
for _, r := range repos {
1452
+
if err := sem.Acquire(ctx, 1); err != nil {
1453
+
log.Errorw("failed to acquire semaphore", "error", err)
1454
+
results <- revCheckResult{err: err}
1455
+
continue
1456
+
}
1452
1457
go func(r comatproto.SyncListRepos_Repo) {
1453
-
if err := sem.Acquire(ctx, 1); err != nil {
1454
-
log.Errorw("failed to acquire semaphore", "error", err)
1455
-
results <- revCheckResult{err: err}
1456
-
return
1457
-
}
1458
1458
defer sem.Release(1)
1459
-
1460
1459
log := log.With("did", r.Did, "remote_rev", r.Rev)
1461
1460
// Fetches the user if we have it, otherwise automatically enqueues it for crawling
1462
1461
ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did)