this repo has no description
feat: commit/checkpoint every 100k
+12
-14
+12
-14
cmd/bsky-users/main.go
+12
-14
cmd/bsky-users/main.go
···
133
133
dbTx.ExecContext(ctx, userTimestampUpdate, did, ts, ts)
134
134
135
135
eventCount += 1
136
-
if eventCount%2500 == 0 {
136
+
if eventCount%100_000 == 0 {
137
137
if err = dbTx.Commit(); err != nil {
138
138
log.Printf("commit failed: %v\n", err)
139
139
} else {
140
140
log.Printf("commit successful\n")
141
141
}
142
142
143
-
if eventCount%25_000 == 0 {
144
-
var results CheckpointResults
145
-
err = dbCnx.QueryRowContext(ctx, "PRAGMA wal_checkpoint(RESTART)").Scan(&results.Blocked, &results.Pages, &results.Transferred)
146
-
switch {
147
-
case err != nil:
148
-
log.Printf("failed checkpoint: %v\n", err)
149
-
case results.Blocked == 1:
150
-
log.Printf("checkpoint: blocked\n")
151
-
case results.Pages == results.Transferred:
152
-
log.Printf("checkpoint: %d pages transferred\n", results.Transferred)
153
-
case results.Pages != results.Transferred:
154
-
log.Printf("checkpoint: %d pages, %d transferred\n", results.Pages, results.Transferred)
155
-
}
143
+
var results CheckpointResults
144
+
err = dbCnx.QueryRowContext(ctx, "PRAGMA wal_checkpoint(RESTART)").Scan(&results.Blocked, &results.Pages, &results.Transferred)
145
+
switch {
146
+
case err != nil:
147
+
log.Printf("failed checkpoint: %v\n", err)
148
+
case results.Blocked == 1:
149
+
log.Printf("checkpoint: blocked\n")
150
+
case results.Pages == results.Transferred:
151
+
log.Printf("checkpoint: %d pages transferred\n", results.Transferred)
152
+
case results.Pages != results.Transferred:
153
+
log.Printf("checkpoint: %d pages, %d transferred\n", results.Pages, results.Transferred)
156
154
}
157
155
158
156
dbTx, err = dbCnx.BeginTx(ctx, nil)