+10 -9

archiver/archiver.go

reviewed

··· 34 34 35 35 type Archiver struct { 36 36 db *gorm.DB 37 37 - slurper *bgs.Slurper 37 37 + Slurper *bgs.Slurper 38 38 didr did.Resolver 39 39 40 40 crawler *CrawlDispatcher ··· 129 129 slOpts.DefaultRepoLimit = config.DefaultRepoLimit 130 130 slOpts.ConcurrencyPerPDS = config.ConcurrencyPerPDS 131 131 slOpts.MaxQueuePerPDS = config.MaxQueuePerPDS 132 132 + slOpts.DefaultRepoLimit = 500_000 132 133 s, err := bgs.NewSlurper(db, arc.handleFedEvent, slOpts) 133 134 if err != nil { 134 135 return nil, err 135 136 } 136 137 137 137 - arc.slurper = s 138 138 + arc.Slurper = s 138 139 139 139 - if err := arc.slurper.RestartAll(); err != nil { 140 140 + if err := arc.Slurper.RestartAll(); err != nil { 140 141 return nil, err 141 142 } 142 143 ··· 205 206 if peering.ID == 0 { 206 207 peering.Host = durl.Host 207 208 peering.SSL = (durl.Scheme == "https") 208 208 - peering.CrawlRateLimit = float64(s.slurper.DefaultCrawlLimit) 209 209 - peering.RateLimit = float64(s.slurper.DefaultPerSecondLimit) 210 210 - peering.HourlyEventLimit = s.slurper.DefaultPerHourLimit 211 211 - peering.DailyEventLimit = s.slurper.DefaultPerDayLimit 212 212 - peering.RepoLimit = s.slurper.DefaultRepoLimit 209 209 + peering.CrawlRateLimit = float64(s.Slurper.DefaultCrawlLimit) 210 210 + peering.RateLimit = float64(s.Slurper.DefaultPerSecondLimit) 211 211 + peering.HourlyEventLimit = s.Slurper.DefaultPerHourLimit 212 212 + peering.DailyEventLimit = s.Slurper.DefaultPerDayLimit 213 213 + peering.RepoLimit = s.Slurper.DefaultRepoLimit 213 214 214 215 if s.ssl && !peering.SSL { 215 216 return nil, fmt.Errorf("did references non-ssl PDS, this is disallowed in prod: %q %q", did, svc.ServiceEndpoint) ··· 722 723 } 723 724 724 725 func (s *Archiver) Shutdown() []error { 725 725 - errs := s.slurper.Shutdown() 726 726 + errs := s.Slurper.Shutdown() 726 727 727 728 if s.crawler != nil { 728 729 s.crawler.Shutdown()

+5

cmd/archit/main.go

reviewed

··· 379 379 arcErr <- err 380 380 }() 381 381 382 382 + time.Sleep(time.Second) 383 383 + if err := arc.Slurper.SubscribeToPds(context.TODO(), "morel.us-east.host.bsky.network", true, true, nil); err != nil { 384 384 + return fmt.Errorf("failed to subscribe to pds: %w", err) 385 385 + } 386 386 + 382 387 slog.Info("startup complete") 383 388 select { 384 389 case <-signals: