julien.rbrt.fr / glean
A social RSS reader built on the AT Protocol. glean.at
glean atproto atmosphere rss feed social app
14
fork

Configure Feed

Select the types of activity you want to include in your feed.

Chunk SQL queries to avoid SQLite parameter limits

Julien Robert 381f2e4d bb615b0c

+42 -16
+42 -16
internal/cluster/social.go
··· 3 3 import ( 4 4 "context" 5 5 "fmt" 6 + "iter" 6 7 ) 7 8 8 9 const maxFollowDepth = 3 10 + 11 + func chunk[T any](s []T, size int) iter.Seq[[]T] { 12 + return func(yield func([]T) bool) { 13 + for i := 0; i < len(s); i += size { 14 + end := i + size 15 + if end > len(s) { 16 + end = len(s) 17 + } 18 + if !yield(s[i:end]) { 19 + return 20 + } 21 + } 22 + } 23 + } 9 24 10 25 type followDistance struct { 11 26 userA string ··· 125 140 } 126 141 defer func() { _ = tx.Rollback() }() 127 142 128 - ph := make([]string, len(dirtyUsers)) 129 - args := make([]any, len(dirtyUsers)) 130 - for i, did := range dirtyUsers { 131 - ph[i] = "?" 132 - args[i] = did 133 - } 134 - if _, err := tx.ExecContext(ctx, 135 - fmt.Sprintf("DELETE FROM recs.follow_distances WHERE user_a IN (%s)", joinPh(ph)), 136 - args..., 137 - ); err != nil { 138 - return err 143 + const sqliteMaxVars = 500 144 + for chunk := range chunk(dirtyUsers, sqliteMaxVars) { 145 + ph := make([]string, len(chunk)) 146 + args := make([]any, len(chunk)) 147 + for i, did := range chunk { 148 + ph[i] = "?" 149 + args[i] = did 150 + } 151 + if _, err := tx.ExecContext(ctx, 152 + fmt.Sprintf("DELETE FROM recs.follow_distances WHERE user_a IN (%s)", joinPh(ph)), 153 + args..., 154 + ); err != nil { 155 + return err 156 + } 139 157 } 140 158 141 159 stmt, err := tx.PrepareContext(ctx, `INSERT INTO recs.follow_distances (user_a, user_b, distance) VALUES (?, ?, ?)`) ··· 150 168 } 151 169 } 152 170 153 - if _, err := tx.ExecContext(ctx, 154 - fmt.Sprintf("UPDATE main.users SET follows_dirty = 0 WHERE did IN (%s)", joinPh(ph)), 155 - args..., 156 - ); err != nil { 157 - return err 171 + for chunk := range chunk(dirtyUsers, sqliteMaxVars) { 172 + ph := make([]string, len(chunk)) 173 + args := make([]any, len(chunk)) 174 + for i, did := range chunk { 175 + ph[i] = "?" 176 + args[i] = did 177 + } 178 + if _, err := tx.ExecContext(ctx, 179 + fmt.Sprintf("UPDATE main.users SET follows_dirty = 0 WHERE did IN (%s)", joinPh(ph)), 180 + args..., 181 + ); err != nil { 182 + return err 183 + } 158 184 } 159 185 160 186 e.logger.Info("follow distances computed", "users", len(dirtyUsers), "pairs", len(distances))