this repo has no description
Tuneable max fetch concurrency
+14
-6
+6
-1
cmd/bigsky/main.go
+6
-1
cmd/bigsky/main.go
···
154
154
Name: "force-dns-udp",
155
155
EnvVars: []string{"FORCE_DNS_UDP"},
156
156
},
157
+
&cli.IntFlag{
158
+
Name: "max-fetch-concurrency",
159
+
Value: 100,
160
+
EnvVars: []string{"MAX_FETCH_CONCURRENCY"},
161
+
},
157
162
}
158
163
159
164
app.Action = Bigsky
···
298
303
299
304
notifman := ¬ifs.NullNotifs{}
300
305
301
-
rf := indexer.NewRepoFetcher(db, repoman)
306
+
rf := indexer.NewRepoFetcher(db, repoman, cctx.Int("max-fetch-concurrency"))
302
307
303
308
ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, rf, true, cctx.Bool("spidering"), cctx.Bool("aggregation"))
304
309
if err != nil {
+1
-1
indexer/indexer.go
+1
-1
indexer/indexer.go
+1
-1
indexer/posts_test.go
+1
-1
indexer/posts_test.go
+4
-1
indexer/repofetch.go
+4
-1
indexer/repofetch.go
···
17
17
"gorm.io/gorm"
18
18
)
19
19
20
-
func NewRepoFetcher(db *gorm.DB, rm *repomgr.RepoManager) *RepoFetcher {
20
+
func NewRepoFetcher(db *gorm.DB, rm *repomgr.RepoManager, maxConcurrency int) *RepoFetcher {
21
21
return &RepoFetcher{
22
22
repoman: rm,
23
23
db: db,
24
24
Limiters: make(map[uint]*rate.Limiter),
25
25
ApplyPDSClientSettings: func(*xrpc.Client) {},
26
+
MaxConcurrency: maxConcurrency,
26
27
}
27
28
}
28
29
···
32
33
33
34
Limiters map[uint]*rate.Limiter
34
35
LimitMux sync.RWMutex
36
+
37
+
MaxConcurrency int
35
38
36
39
ApplyPDSClientSettings func(*xrpc.Client)
37
40
}
+1
-1
pds/server.go
+1
-1
pds/server.go
···
76
76
repoman := repomgr.NewRepoManager(cs, kmgr)
77
77
notifman := notifs.NewNotificationManager(db, repoman.GetRecord)
78
78
79
-
rf := indexer.NewRepoFetcher(db, repoman)
79
+
rf := indexer.NewRepoFetcher(db, repoman, 10)
80
80
81
81
ix, err := indexer.NewIndexer(db, notifman, evtman, didr, rf, false, true, true)
82
82
if err != nil {
+1
-1
testing/utils.go
+1
-1
testing/utils.go
···
488
488
diskpersist, err := events.NewDiskPersistence(filepath.Join(dir, "dp-primary"), filepath.Join(dir, "dp-archive"), maindb, opts)
489
489
490
490
evtman := events.NewEventManager(diskpersist)
491
-
rf := indexer.NewRepoFetcher(maindb, repoman)
491
+
rf := indexer.NewRepoFetcher(maindb, repoman, 10)
492
492
493
493
ix, err := indexer.NewIndexer(maindb, notifman, evtman, didr, rf, true, true, true)
494
494
if err != nil {