this repo has no description
Per-day new PDS limiter to allow less constrained federation
+38
-4
+32
-4
bgs/fedmgr.go
+32
-4
bgs/fedmgr.go
···
42
42
DefaultCrawlLimit rate.Limit
43
43
DefaultRepoLimit int64
44
44
45
+
NewPDSPerDayLimiter *slidingwindow.Limiter
46
+
45
47
newSubsDisabled bool
46
48
trustedDomains []string
47
49
···
222
224
s.newSubsDisabled = sc.NewSubsDisabled
223
225
s.trustedDomains = sc.TrustedDomains
224
226
227
+
s.NewPDSPerDayLimiter, _ = slidingwindow.NewLimiter(time.Hour*24, sc.NewPDSPerDayLimit, windowFunc)
228
+
225
229
return nil
226
230
}
227
231
228
232
type SlurpConfig struct {
229
233
gorm.Model
230
234
231
-
NewSubsDisabled bool
232
-
TrustedDomains pq.StringArray `gorm:"type:text[]"`
235
+
NewSubsDisabled bool
236
+
TrustedDomains pq.StringArray `gorm:"type:text[]"`
237
+
NewPDSPerDayLimit int64
233
238
}
234
239
235
240
func (s *Slurper) SetNewSubsDisabled(dis bool) error {
···
250
255
return s.newSubsDisabled
251
256
}
252
257
258
+
func (s *Slurper) SetNewPDSPerDayLimit(limit int64) error {
259
+
s.lk.Lock()
260
+
defer s.lk.Unlock()
261
+
262
+
if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("new_pds_per_day_limit", limit).Error; err != nil {
263
+
return err
264
+
}
265
+
266
+
s.NewPDSPerDayLimiter.SetLimit(limit)
267
+
return nil
268
+
}
269
+
270
+
func (s *Slurper) GetNewPDSPerDayLimit() int64 {
271
+
s.lk.Lock()
272
+
defer s.lk.Unlock()
273
+
return s.NewPDSPerDayLimiter.Limit()
274
+
}
275
+
253
276
func (s *Slurper) AddTrustedDomain(domain string) error {
254
277
s.lk.Lock()
255
278
defer s.lk.Unlock()
···
306
329
// Checks whether a host is allowed to be subscribed to
307
330
// must be called with the slurper lock held
308
331
func (s *Slurper) canSlurpHost(host string) bool {
332
+
// Check if we're over the limit for new PDSs today
333
+
if !s.NewPDSPerDayLimiter.Allow() {
334
+
return false
335
+
}
336
+
309
337
// Check if the host is a trusted domain
310
338
for _, d := range s.trustedDomains {
311
339
// If the domain starts with a *., it's a wildcard
···
324
352
return !s.newSubsDisabled
325
353
}
326
354
327
-
func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, overrideTrustList bool) error {
355
+
func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, adminOverride bool) error {
328
356
// TODO: for performance, lock on the hostname instead of global
329
357
s.lk.Lock()
330
358
defer s.lk.Unlock()
···
344
372
}
345
373
346
374
if peering.ID == 0 {
347
-
if !overrideTrustList && !s.canSlurpHost(host) {
375
+
if !adminOverride && !s.canSlurpHost(host) {
348
376
return ErrNewSubsDisabled
349
377
}
350
378
// New PDS!
+6
cmd/palomar/docker-compose.yml
+6
cmd/palomar/docker-compose.yml
···
14
14
- "plugins.security.disabled=true"
15
15
- "bootstrap.memory_lock=true" # Disable JVM heap memory swapping
16
16
- "OPENSEARCH_JAVA_OPTS=-Xms4096m -Xmx4096m" # Set min and max JVM heap sizes to at least 50% of system RAM
17
+
- "OPENSEARCH_INITIAL_ADMIN_PASSWORD=0penSearch-Pal0mar"
17
18
ulimits:
18
19
memlock:
19
20
soft: -1
···
43
44
- "PALOMAR_BGS_SYNC_RATE_LIMIT=20"
44
45
- "PALOMAR_INDEX_MAX_CONCURRENCY=5"
45
46
- "DATABASE_URL=sqlite:///data/palomar/search.db"
47
+
- "PALOMAR_BIND=:3997"
48
+
- "PALOMAR_METRICS_LISTEN=:3996"
46
49
depends_on:
47
50
- opensearch
51
+
ports:
52
+
- "3997:3997"
53
+
- "3996:3996"
48
54
volumes:
49
55
- type: bind
50
56
source: ../../data