this repo has no description
Bump gorm dependencies, seeing some weird sqlite stuff w/ large DBs (#652)
+22
-8
+19
-5
backfill/gormstore.go
+19
-5
backfill/gormstore.go
···
34
34
type GormDBJob struct {
35
35
gorm.Model
36
36
Repo string `gorm:"unique;index"`
37
-
State string `gorm:"index;index:failed_job_idx,where:state like 'failed%'"`
37
+
State string `gorm:"index:enqueued_job_idx,where:state = 'enqueued';index:retryable_job_idx,where:state like 'failed%'"`
38
38
Rev string
39
39
RetryCount int
40
-
RetryAfter *time.Time `gorm:"index"`
40
+
RetryAfter *time.Time `gorm:"index:retryable_job_idx,sort:desc"`
41
41
}
42
42
43
43
// Gormstore is a gorm-backed implementation of the Backfill Store interface
···
66
66
67
67
func (s *Gormstore) loadJobs(ctx context.Context, limit int) error {
68
68
var todo []string
69
-
if err := s.db.Model(GormDBJob{}).Limit(limit).Select("repo").
70
-
Where("state = 'enqueued' OR (state like 'failed%' AND (retry_after = NULL OR retry_after < ?))", time.Now()).Scan(&todo).Error; err != nil {
69
+
if err := s.db.Raw(
70
+
`SELECT repo FROM gorm_db_jobs INDEXED BY enqueued_job_idx WHERE state = 'enqueued' LIMIT ?`,
71
+
limit,
72
+
).Scan(&todo).Error; err != nil {
71
73
return err
74
+
}
75
+
76
+
if len(todo) < limit {
77
+
var moreTodo []string
78
+
if err := s.db.Raw(
79
+
`SELECT repo FROM gorm_db_jobs INDEXED BY retryable_job_idx WHERE state like 'failed%' AND (retry_after = NULL OR retry_after < ?) LIMIT ?`,
80
+
time.Now(),
81
+
limit-len(todo),
82
+
).Scan(&moreTodo).Error; err != nil {
83
+
return err
84
+
}
85
+
todo = append(todo, moreTodo...)
72
86
}
73
87
74
88
s.taskQueue = append(s.taskQueue, todo...)
···
296
310
j.updatedAt = time.Now()
297
311
298
312
// Persist the job to the database
299
-
j.dbj.Rev = r
313
+
300
314
return j.db.Save(j.dbj).Error
301
315
}
302
316
+1
-1
go.mod
+1
-1
go.mod
+2
-2
go.sum
+2
-2
go.sum
···
1079
1079
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
1080
1080
gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U=
1081
1081
gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A=
1082
-
gorm.io/driver/sqlite v1.5.4 h1:IqXwXi8M/ZlPzH/947tn5uik3aYQslP9BVveoax0nV0=
1083
-
gorm.io/driver/sqlite v1.5.4/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
1082
+
gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E=
1083
+
gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE=
1084
1084
gorm.io/gorm v1.24.7-0.20230306060331-85eaf9eeda11/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
1085
1085
gorm.io/gorm v1.25.9 h1:wct0gxZIELDk8+ZqF/MVnHLkA1rvYlBWUMv2EdsK1g8=
1086
1086
gorm.io/gorm v1.25.9/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=