this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

break up requests for block refs (#334)

turns out some repos have *a lot* of shards and this makes sql unhappy.

authored by

Whyrusleeping and committed by
GitHub
0499e8a6 976cb2bc

+26 -3
+26 -3
carstore/bs.go
··· 1151 1151 return targets, nil 1152 1152 } 1153 1153 1154 + func (cs *CarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) ([]blockRef, error) { 1155 + ctx, span := otel.Tracer("carstore").Start(ctx, "getBlockRefsForShards") 1156 + defer span.End() 1157 + 1158 + chunkSize := 10000 1159 + out := make([]blockRef, 0, len(shardIds)) 1160 + for i := 0; i < len(shardIds); i += chunkSize { 1161 + sl := shardIds[i:] 1162 + if len(sl) > chunkSize { 1163 + sl = sl[:chunkSize] 1164 + } 1165 + 1166 + var brefs []blockRef 1167 + if err := cs.meta.Raw(`select * from block_refs where shard in (?)`, sl).Scan(&brefs).Error; err != nil { 1168 + return nil, err 1169 + } 1170 + 1171 + out = append(out, brefs...) 1172 + } 1173 + 1174 + return out, nil 1175 + } 1176 + 1154 1177 type CompactionStats struct { 1155 1178 StartShards int `json:"startShards"` 1156 1179 NewShards int `json:"newShards"` ··· 1180 1203 shardsById[s.ID] = s 1181 1204 } 1182 1205 1183 - var brefs []blockRef 1184 - if err := cs.meta.WithContext(ctx).Raw(`select shard, cid from block_refs where shard in (?)`, shardIds).Scan(&brefs).Error; err != nil { 1185 - return nil, err 1206 + brefs, err := cs.getBlockRefsForShards(ctx, shardIds) 1207 + if err != nil { 1208 + return nil, fmt.Errorf("getting block refs failed: %w", err) 1186 1209 } 1187 1210 1188 1211 var staleRefs []staleRef