this repo has no description
some extra tweaks required for fully handling 'too big' slices (#72)
The last PR handled this for playback but missed catching when they
happen on the fly, also dropped some log warnings to infos so I could
see what was even going wrong.
authored by
Whyrusleeping
and committed by
GitHub
1d2c5c4c
2099cb45
+19
-11
+17
-7
indexer/indexer.go
+17
-7
indexer/indexer.go
···
10
10
11
11
atproto "github.com/bluesky-social/indigo/api/atproto"
12
12
bsky "github.com/bluesky-social/indigo/api/bsky"
13
+
"github.com/bluesky-social/indigo/carstore"
13
14
"github.com/bluesky-social/indigo/events"
14
15
"github.com/bluesky-social/indigo/models"
15
16
"github.com/bluesky-social/indigo/notifs"
···
137
138
prevstr = &s
138
139
}
139
140
141
+
toobig := false
142
+
slice := evt.RepoSlice
143
+
if len(slice) > carstore.MaxSliceLength {
144
+
slice = nil
145
+
toobig = true
146
+
147
+
}
148
+
140
149
log.Infow("Sending event", "did", did)
141
150
if err := ix.events.AddEvent(ctx, &events.XRPCStreamEvent{
142
151
RepoAppend: &events.RepoAppend{
143
152
Repo: did,
144
153
Prev: prevstr,
145
-
Blocks: evt.RepoSlice,
154
+
Blocks: slice,
146
155
Commit: evt.NewRoot.String(),
147
156
Time: time.Now().Format(util.ISO8601),
148
157
Ops: outops,
158
+
TooBig: toobig,
149
159
},
150
160
PrivUid: evt.User,
151
161
}); err != nil {
···
323
333
if e.Type == "mention" {
324
334
_, err := ix.GetUserOrMissing(ctx, e.Value)
325
335
if err != nil {
326
-
log.Warnw("failed to parse user mention", "ref", e.Value, "err", err)
336
+
log.Infow("failed to parse user mention", "ref", e.Value, "err", err)
327
337
}
328
338
}
329
339
}
···
331
341
if rec.Reply != nil {
332
342
if rec.Reply.Parent != nil {
333
343
if err := ix.crawlAtUriRef(ctx, rec.Reply.Parent.Uri); err != nil {
334
-
log.Warnw("failed to crawl reply parent", "cid", op.RecCid, "replyuri", rec.Reply.Parent.Uri, "err", err)
344
+
log.Infow("failed to crawl reply parent", "cid", op.RecCid, "replyuri", rec.Reply.Parent.Uri, "err", err)
335
345
}
336
346
}
337
347
338
348
if rec.Reply.Root != nil {
339
349
if err := ix.crawlAtUriRef(ctx, rec.Reply.Root.Uri); err != nil {
340
-
log.Warnw("failed to crawl reply root", "cid", op.RecCid, "rooturi", rec.Reply.Root.Uri, "err", err)
350
+
log.Infow("failed to crawl reply root", "cid", op.RecCid, "rooturi", rec.Reply.Root.Uri, "err", err)
341
351
}
342
352
}
343
353
}
···
346
356
case *bsky.FeedRepost:
347
357
if rec.Subject != nil {
348
358
if err := ix.crawlAtUriRef(ctx, rec.Subject.Uri); err != nil {
349
-
log.Warnw("failed to crawl repost subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err)
359
+
log.Infow("failed to crawl repost subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err)
350
360
}
351
361
}
352
362
return nil
353
363
case *bsky.FeedVote:
354
364
if rec.Subject != nil {
355
365
if err := ix.crawlAtUriRef(ctx, rec.Subject.Uri); err != nil {
356
-
log.Warnw("failed to crawl vote subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err)
366
+
log.Infow("failed to crawl vote subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err)
357
367
}
358
368
}
359
369
return nil
···
361
371
if rec.Subject != nil {
362
372
_, err := ix.GetUserOrMissing(ctx, rec.Subject.Did)
363
373
if err != nil {
364
-
log.Warnw("failed to crawl follow subject", "cid", op.RecCid, "subjectdid", rec.Subject.Did, "err", err)
374
+
log.Infow("failed to crawl follow subject", "cid", op.RecCid, "subjectdid", rec.Subject.Did, "err", err)
365
375
}
366
376
}
367
377
return nil
+2
-4
repomgr/repomgr.go
+2
-4
repomgr/repomgr.go
···
854
854
if err != nil {
855
855
return err
856
856
}
857
+
857
858
if len(slice) > carstore.MaxSliceLength {
858
-
// TODO: this should never happen because the same check exists
859
-
// inside the carstore, possibly remove this error later (assuming
860
-
// we never see it again)
861
-
return fmt.Errorf("(CRITICAL) resultant slice was too large, len=%d user=%d old=%s new=%s", len(slice), user, old, nu)
859
+
log.Warnw("output slice too large", "len", len(slice), "user", user, "old", old, "new", nu)
862
860
}
863
861
864
862
if err := rm.updateUserRepoHead(ctx, user, nu); err != nil {