this repo has no description
Make BGS spidering optional (#396)
+16
-8
+6
-1
cmd/bigsky/main.go
+6
-1
cmd/bigsky/main.go
···
100
100
Name: "aggregation",
101
101
Value: false,
102
102
},
103
+
&cli.BoolFlag{
104
+
Name: "spidering",
105
+
Value: true,
106
+
EnvVars: []string{"BGS_SPIDERING"},
107
+
},
103
108
&cli.StringFlag{
104
109
Name: "api-listen",
105
110
Value: ":2470",
···
278
283
279
284
notifman := ¬ifs.NullNotifs{}
280
285
281
-
ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, repoman, true, cctx.Bool("aggregation"))
286
+
ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, repoman, true, cctx.Bool("spidering"), cctx.Bool("aggregation"))
282
287
if err != nil {
283
288
return err
284
289
}
+7
-4
indexer/indexer.go
+7
-4
indexer/indexer.go
···
51
51
LimitMux sync.RWMutex
52
52
53
53
doAggregations bool
54
+
doSpider bool
54
55
55
56
SendRemoteFollow func(context.Context, string, uint) error
56
57
CreateExternalUser func(context.Context, string) (*models.ActorInfo, error)
57
58
ApplyPDSClientSettings func(*xrpc.Client)
58
59
}
59
60
60
-
func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events.EventManager, didr did.Resolver, repoman *repomgr.RepoManager, crawl, aggregate bool) (*Indexer, error) {
61
+
func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events.EventManager, didr did.Resolver, repoman *repomgr.RepoManager, crawl, aggregate, spider bool) (*Indexer, error) {
61
62
db.AutoMigrate(&models.FeedPost{})
62
63
db.AutoMigrate(&models.ActorInfo{})
63
64
db.AutoMigrate(&models.FollowRecord{})
···
72
73
didr: didr,
73
74
Limiters: make(map[uint]*rate.Limiter),
74
75
doAggregations: aggregate,
76
+
doSpider: spider,
75
77
SendRemoteFollow: func(context.Context, string, uint) error {
76
78
return nil
77
79
},
···
181
183
return fmt.Errorf("handle recordCreate: %w", err)
182
184
}
183
185
}
184
-
if err := ix.crawlRecordReferences(ctx, op); err != nil {
185
-
return err
186
+
if ix.doSpider {
187
+
if err := ix.crawlRecordReferences(ctx, op); err != nil {
188
+
return err
189
+
}
186
190
}
187
-
188
191
case repomgr.EvtKindDeleteRecord:
189
192
if ix.doAggregations {
190
193
if err := ix.handleRecordDelete(ctx, evt, op, true); err != nil {
+1
-1
indexer/posts_test.go
+1
-1
indexer/posts_test.go
+1
-1
pds/server.go
+1
-1
pds/server.go
···
80
80
repoman := repomgr.NewRepoManager(cs, kmgr)
81
81
notifman := notifs.NewNotificationManager(db, repoman.GetRecord)
82
82
83
-
ix, err := indexer.NewIndexer(db, notifman, evtman, didr, repoman, false, true)
83
+
ix, err := indexer.NewIndexer(db, notifman, evtman, didr, repoman, false, true, true)
84
84
if err != nil {
85
85
return nil, err
86
86
}
+1
-1
testing/utils.go
+1
-1
testing/utils.go
···
436
436
437
437
evtman := events.NewEventManager(diskpersist)
438
438
439
-
ix, err := indexer.NewIndexer(maindb, notifman, evtman, didr, repoman, true, true)
439
+
ix, err := indexer.NewIndexer(maindb, notifman, evtman, didr, repoman, true, true, true)
440
440
if err != nil {
441
441
return nil, err
442
442
}