this repo has no description
break up requests for block refs (#334)
turns out some repos have *a lot* of shards and this makes sql unhappy.
authored by
Whyrusleeping
and committed by
GitHub
0499e8a6
976cb2bc
+26
-3
+26
-3
carstore/bs.go
+26
-3
carstore/bs.go
···
1151
1151
return targets, nil
1152
1152
}
1153
1153
1154
+
func (cs *CarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) ([]blockRef, error) {
1155
+
ctx, span := otel.Tracer("carstore").Start(ctx, "getBlockRefsForShards")
1156
+
defer span.End()
1157
+
1158
+
chunkSize := 10000
1159
+
out := make([]blockRef, 0, len(shardIds))
1160
+
for i := 0; i < len(shardIds); i += chunkSize {
1161
+
sl := shardIds[i:]
1162
+
if len(sl) > chunkSize {
1163
+
sl = sl[:chunkSize]
1164
+
}
1165
+
1166
+
var brefs []blockRef
1167
+
if err := cs.meta.Raw(`select * from block_refs where shard in (?)`, sl).Scan(&brefs).Error; err != nil {
1168
+
return nil, err
1169
+
}
1170
+
1171
+
out = append(out, brefs...)
1172
+
}
1173
+
1174
+
return out, nil
1175
+
}
1176
+
1154
1177
type CompactionStats struct {
1155
1178
StartShards int `json:"startShards"`
1156
1179
NewShards int `json:"newShards"`
···
1180
1203
shardsById[s.ID] = s
1181
1204
}
1182
1205
1183
-
var brefs []blockRef
1184
-
if err := cs.meta.WithContext(ctx).Raw(`select shard, cid from block_refs where shard in (?)`, shardIds).Scan(&brefs).Error; err != nil {
1185
-
return nil, err
1206
+
brefs, err := cs.getBlockRefsForShards(ctx, shardIds)
1207
+
if err != nil {
1208
+
return nil, fmt.Errorf("getting block refs failed: %w", err)
1186
1209
}
1187
1210
1188
1211
var staleRefs []staleRef