this repo has no description
make compaction worker count configurable (#803)
Want to be able to tweak this manually so we can turn it higher during
off-peak if necessary
authored by
Whyrusleeping
and committed by
GitHub
ca65ff25
1f22a51a
+21
-11
+15
-11
bgs/bgs.go
+15
-11
bgs/bgs.go
···
107
107
}
108
108
109
109
type BGSConfig struct {
110
-
SSL bool
111
-
CompactInterval time.Duration
112
-
DefaultRepoLimit int64
113
-
ConcurrencyPerPDS int64
114
-
MaxQueuePerPDS int64
110
+
SSL bool
111
+
CompactInterval time.Duration
112
+
DefaultRepoLimit int64
113
+
ConcurrencyPerPDS int64
114
+
MaxQueuePerPDS int64
115
+
NumCompactionWorkers int
115
116
}
116
117
117
118
func DefaultBGSConfig() *BGSConfig {
118
119
return &BGSConfig{
119
-
SSL: true,
120
-
CompactInterval: 4 * time.Hour,
121
-
DefaultRepoLimit: 100,
122
-
ConcurrencyPerPDS: 100,
123
-
MaxQueuePerPDS: 1_000,
120
+
SSL: true,
121
+
CompactInterval: 4 * time.Hour,
122
+
DefaultRepoLimit: 100,
123
+
ConcurrencyPerPDS: 100,
124
+
MaxQueuePerPDS: 1_000,
125
+
NumCompactionWorkers: 2,
124
126
}
125
127
}
126
128
···
168
170
return nil, err
169
171
}
170
172
171
-
compactor := NewCompactor(nil)
173
+
cOpts := DefaultCompactorOptions()
174
+
cOpts.NumWorkers = config.NumCompactionWorkers
175
+
compactor := NewCompactor(cOpts)
172
176
compactor.requeueInterval = config.CompactInterval
173
177
compactor.Start(bgs)
174
178
bgs.compactor = compactor
+6
cmd/bigsky/main.go
+6
cmd/bigsky/main.go
···
195
195
EnvVars: []string{"RELAY_EVENT_PLAYBACK_TTL"},
196
196
Value: 72 * time.Hour,
197
197
},
198
+
&cli.IntFlag{
199
+
Name: "num-compaction-workers",
200
+
EnvVars: []string{"RELAY_NUM_COMPACTION_WORKERS"},
201
+
Value: 2,
202
+
},
198
203
}
199
204
200
205
app.Action = runBigsky
···
413
418
bgsConfig.ConcurrencyPerPDS = cctx.Int64("concurrency-per-pds")
414
419
bgsConfig.MaxQueuePerPDS = cctx.Int64("max-queue-per-pds")
415
420
bgsConfig.DefaultRepoLimit = cctx.Int64("default-repo-limit")
421
+
bgsConfig.NumCompactionWorkers = cctx.Int("num-compaction-workers")
416
422
bgs, err := libbgs.NewBGS(db, ix, repoman, evtman, cachedidr, rf, hr, bgsConfig)
417
423
if err != nil {
418
424
return err