+17

.tangled/workflows/test.yml

reviewed

··· 8 8 nixpkgs: 9 9 - go 10 10 - gcc 11 11 + - postgresql 12 12 + - shadow 13 13 + - util-linux 11 14 12 15 steps: 13 16 - name: patch static dir 14 17 command: | 15 18 mkdir -p appview/pages/static; touch appview/pages/static/x 16 19 20 20 + - name: start postgres 21 21 + command: | 22 22 + set -euo pipefail 23 23 + mkdir -p /pg/socket 24 24 + useradd -r -m -d /pg/home pguser 25 25 + chown -R pguser /pg 26 26 + uid=$(id -u pguser); gid=$(id -g pguser) 27 27 + setpriv --reuid "$uid" --regid "$gid" --init-groups -- initdb -D /pg/home/data --auth=trust --username=postgres 28 28 + setpriv --reuid "$uid" --regid "$gid" --init-groups -- pg_ctl -D /pg/home/data -l /pg/home/pg.log -o "-p 5432 -k /pg/socket -h 127.0.0.1" start 29 29 + for i in $(seq 1 20); do pg_isready -h 127.0.0.1 -p 5432 -U postgres && break; sleep 0.5; done 30 30 + pg_isready -h 127.0.0.1 -p 5432 -U postgres 31 31 + createdb -h 127.0.0.1 -p 5432 -U postgres mirror 32 32 + 17 33 - name: run linter 18 34 environment: 19 35 CGO_ENABLED: 1 ··· 23 39 - name: run all tests 24 40 environment: 25 41 CGO_ENABLED: 1 42 42 + TEST_POSTGRES_URL: postgresql://postgres@127.0.0.1:5432/mirror?sslmode=disable 26 43 command: | 27 44 go test -v ./...

+29 -7

go.mod

reviewed

··· 63 63 ) 64 64 65 65 require ( 66 66 - dario.cat/mergo v1.0.1 // indirect 66 66 + dario.cat/mergo v1.0.2 // indirect 67 67 + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect 67 68 github.com/BurntSushi/toml v0.3.1 // indirect 68 69 github.com/Microsoft/go-winio v0.6.2 // indirect 69 70 github.com/ProtonMail/go-crypto v1.3.0 // indirect ··· 120 121 github.com/containerd/errdefs v1.0.0 // indirect 121 122 github.com/containerd/errdefs/pkg v0.3.0 // indirect 122 123 github.com/containerd/log v0.1.0 // indirect 124 124 + github.com/containerd/platforms v0.2.1 // indirect 125 125 + github.com/cpuguy83/dockercfg v0.3.2 // indirect 123 126 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 124 127 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 125 128 github.com/distribution/reference v0.6.0 // indirect 126 129 github.com/dlclark/regexp2 v1.11.5 // indirect 127 127 - github.com/docker/go-connections v0.5.0 // indirect 130 130 + github.com/docker/go-connections v0.6.0 // indirect 128 131 github.com/docker/go-units v0.5.0 // indirect 129 132 github.com/earthboundkid/versioninfo/v2 v2.24.1 // indirect 133 133 + github.com/ebitengine/purego v0.10.0 // indirect 130 134 github.com/emirpasic/gods v1.18.1 // indirect 131 135 github.com/felixge/httpsnoop v1.0.4 // indirect 132 136 github.com/fsnotify/fsnotify v1.6.0 // indirect ··· 137 141 github.com/go-logfmt/logfmt v0.6.0 // indirect 138 142 github.com/go-logr/logr v1.4.3 // indirect 139 143 github.com/go-logr/stdr v1.2.2 // indirect 144 144 + github.com/go-ole/go-ole v1.2.6 // indirect 140 145 github.com/go-redis/cache/v9 v9.0.0 // indirect 141 146 github.com/go-test/deep v1.1.1 // indirect 142 147 github.com/goccy/go-json v0.10.5 // indirect ··· 176 181 github.com/jackc/puddle/v2 v2.2.2 // indirect 177 182 github.com/json-iterator/go v1.1.12 // indirect 178 183 github.com/kevinburke/ssh_config v1.2.0 // indirect 179 179 - github.com/klauspost/compress v1.18.0 // indirect 184 184 + github.com/klauspost/compress v1.18.5 // indirect 180 185 github.com/klauspost/cpuid/v2 v2.3.0 // indirect 181 186 github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 187 187 + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect 188 188 + github.com/magiconair/properties v1.8.10 // indirect 182 189 github.com/mattn/go-isatty v0.0.20 // indirect 183 190 github.com/mattn/go-runewidth v0.0.16 // indirect 184 191 github.com/minio/sha256-simd v1.0.1 // indirect 185 192 github.com/mitchellh/mapstructure v1.5.0 // indirect 186 193 github.com/moby/docker-image-spec v1.3.1 // indirect 194 194 + github.com/moby/go-archive v0.2.0 // indirect 195 195 + github.com/moby/moby/api v1.54.1 // indirect 196 196 + github.com/moby/moby/client v0.4.0 // indirect 197 197 + github.com/moby/patternmatcher v0.6.1 // indirect 187 198 github.com/moby/sys/atomicwriter v0.1.0 // indirect 199 199 + github.com/moby/sys/sequential v0.6.0 // indirect 200 200 + github.com/moby/sys/user v0.4.0 // indirect 201 201 + github.com/moby/sys/userns v0.1.0 // indirect 188 202 github.com/moby/term v0.5.2 // indirect 189 203 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 190 204 github.com/modern-go/reflect2 v1.0.2 // indirect ··· 206 220 github.com/pkg/errors v0.9.1 // indirect 207 221 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 208 222 github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect 223 223 + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect 209 224 github.com/prometheus/client_model v0.6.2 // indirect 210 225 github.com/prometheus/common v0.67.5 // indirect 211 226 github.com/prometheus/procfs v0.19.2 // indirect 212 227 github.com/rivo/uniseg v0.4.7 // indirect 213 228 github.com/ryanuber/go-glob v1.0.0 // indirect 214 229 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect 230 230 + github.com/shirou/gopsutil/v4 v4.26.3 // indirect 231 231 + github.com/sirupsen/logrus v1.9.4 // indirect 215 232 github.com/spaolacci/murmur3 v1.1.0 // indirect 233 233 + github.com/testcontainers/testcontainers-go v0.42.0 // indirect 234 234 + github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 // indirect 216 235 github.com/tidwall/gjson v1.18.0 // indirect 217 236 github.com/tidwall/match v1.2.0 // indirect 218 237 github.com/tidwall/pretty v1.2.1 // indirect 219 238 github.com/tidwall/sjson v1.2.5 // indirect 239 239 + github.com/tklauser/go-sysconf v0.3.16 // indirect 240 240 + github.com/tklauser/numcpus v0.11.0 // indirect 220 241 github.com/vmihailenco/go-tinylfu v0.2.2 // indirect 221 242 github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect 222 243 github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 223 244 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect 245 245 + github.com/yusufpapurcu/wmi v1.2.4 // indirect 224 246 gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect 225 247 gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect 226 248 go.etcd.io/bbolt v1.4.0 // indirect 227 249 go.opentelemetry.io/auto/sdk v1.2.1 // indirect 228 250 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect 229 229 - go.opentelemetry.io/otel v1.40.0 // indirect 251 251 + go.opentelemetry.io/otel v1.41.0 // indirect 230 252 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect 231 231 - go.opentelemetry.io/otel/metric v1.40.0 // indirect 232 232 - go.opentelemetry.io/otel/trace v1.40.0 // indirect 253 253 + go.opentelemetry.io/otel/metric v1.41.0 // indirect 254 254 + go.opentelemetry.io/otel/trace v1.41.0 // indirect 233 255 go.opentelemetry.io/proto/otlp v1.9.0 // indirect 234 256 go.uber.org/atomic v1.11.0 // indirect 235 257 go.uber.org/multierr v1.11.0 // indirect ··· 237 259 go.yaml.in/yaml/v2 v2.4.3 // indirect 238 260 golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect 239 261 golang.org/x/sync v0.19.0 // indirect 240 240 - golang.org/x/sys v0.41.0 // indirect 262 262 + golang.org/x/sys v0.42.0 // indirect 241 263 golang.org/x/text v0.34.0 // indirect 242 264 golang.org/x/time v0.12.0 // indirect 243 265 google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect

+58

go.sum

reviewed

··· 1 1 dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= 2 2 dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= 3 3 + dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= 4 4 + dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= 3 5 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= 4 6 github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 5 7 github.com/Blank-Xu/sql-adapter v1.1.1 h1:+g7QXU9sl/qT6Po97teMpf3GjAO0X9aFaqgSePXvYko= ··· 191 193 github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= 192 194 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= 193 195 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= 196 196 + github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= 197 197 + github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= 198 198 + github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= 199 199 + github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= 194 200 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 195 201 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 196 202 github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= ··· 218 224 github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 219 225 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= 220 226 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= 227 227 + github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= 228 228 + github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= 221 229 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= 222 230 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 223 231 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 224 232 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 225 233 github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg= 226 234 github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw= 235 235 + github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= 236 236 + github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= 227 237 github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= 228 238 github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= 229 239 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= ··· 264 274 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 265 275 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 266 276 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 277 277 + github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= 278 278 + github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= 267 279 github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= 268 280 github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= 269 281 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= ··· 307 319 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 308 320 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 309 321 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 322 322 + github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 310 323 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 311 324 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 312 325 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= ··· 413 426 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 414 427 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 415 428 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 429 429 + github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= 430 430 + github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= 416 431 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= 417 432 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= 418 433 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= ··· 427 442 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 428 443 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 429 444 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 445 445 + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= 446 446 + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= 447 447 + github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= 448 448 + github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 430 449 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 431 450 github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 432 451 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= ··· 443 462 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 444 463 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= 445 464 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= 465 465 + github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= 466 466 + github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= 467 467 + github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4= 468 468 + github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= 469 469 + github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= 470 470 + github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= 471 471 + github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= 472 472 + github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= 446 473 github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= 447 474 github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= 448 475 github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= 449 476 github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= 477 477 + github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= 478 478 + github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= 479 479 + github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= 480 480 + github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= 450 481 github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= 451 482 github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= 452 483 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= ··· 526 557 github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= 527 558 github.com/posthog/posthog-go v1.5.5 h1:2o3j7IrHbTIfxRtj4MPaXKeimuTYg49onNzNBZbwksM= 528 559 github.com/posthog/posthog-go v1.5.5/go.mod h1:3RqUmSnPuwmeVj/GYrS75wNGqcAKdpODiwc83xZWgdE= 560 560 + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= 561 561 + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= 529 562 github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= 530 563 github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= 531 564 github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= ··· 553 586 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 554 587 github.com/sethvargo/go-envconfig v1.1.0 h1:cWZiJxeTm7AlCvzGXrEXaSTCNgip5oJepekh/BOQuog= 555 588 github.com/sethvargo/go-envconfig v1.1.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw= 589 589 + github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= 590 590 + github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= 556 591 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 557 592 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 558 593 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 594 594 + github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= 595 595 + github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= 559 596 github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= 560 597 github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= 561 598 github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= ··· 579 616 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 580 617 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 581 618 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 619 619 + github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= 620 620 + github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= 621 621 + github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo= 622 622 + github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs= 582 623 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 583 624 github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= 584 625 github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= ··· 590 631 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 591 632 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= 592 633 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= 634 634 + github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= 635 635 + github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= 636 636 + github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= 637 637 + github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= 593 638 github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 594 639 github.com/urfave/cli/v3 v3.6.2 h1:lQuqiPrZ1cIz8hz+HcrG0TNZFxU70dPZ3Yl+pSrH9A8= 595 640 github.com/urfave/cli/v3 v3.6.2/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= ··· 618 663 github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= 619 664 github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc h1:+IAOyRda+RLrxa1WC7umKOZRsGq4QrFFMYApOeHzQwQ= 620 665 github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc/go.mod h1:ovIvrum6DQJA4QsJSovrkC4saKHQVs7TvcaeO8AIl5I= 666 666 + github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= 667 667 + github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= 621 668 gitlab.com/staticnoise/goldmark-callout v0.0.0-20240609120641-6366b799e4ab h1:gK9tS6QJw5F0SIhYJnGG2P83kuabOdmWBbSmZhJkz2A= 622 669 gitlab.com/staticnoise/goldmark-callout v0.0.0-20240609120641-6366b799e4ab/go.mod h1:SPu13/NPe1kMrbGoJldQwqtpNhXsmIuHCfm/aaGjU0c= 623 670 gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA= ··· 634 681 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= 635 682 go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= 636 683 go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= 684 684 + go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= 685 685 + go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= 637 686 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= 638 687 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= 639 688 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= 640 689 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= 641 690 go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= 642 691 go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= 692 692 + go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= 693 693 + go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= 643 694 go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= 644 695 go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= 645 696 go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= 646 697 go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= 647 698 go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= 648 699 go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= 700 700 + go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= 701 701 + go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= 649 702 go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= 650 703 go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= 651 704 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= ··· 722 775 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 723 776 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 724 777 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 778 778 + golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 725 779 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 726 780 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 727 781 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 728 782 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 729 783 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 730 784 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 785 785 + golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 731 786 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 732 787 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 733 788 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 734 789 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 735 790 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 791 791 + golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 736 792 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 737 793 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 738 794 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= ··· 750 806 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 751 807 golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= 752 808 golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 809 809 + golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= 810 810 + golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= 753 811 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 754 812 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 755 813 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

+37

knotmirror/db/repos.go

reviewed

··· 67 67 return nil 68 68 } 69 69 70 70 + func MarkDesynchronized(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey) error { 71 71 + if _, err := e.ExecContext(ctx, 72 72 + `update repos 73 73 + set state = $1 74 74 + where did = $2 and rkey = $3 and state in ($4, $5, $6, $7)`, 75 75 + models.RepoStateDesynchronized, 76 76 + did, rkey, 77 77 + models.RepoStateActive, 78 78 + models.RepoStateDesynchronized, 79 79 + models.RepoStateResyncing, 80 80 + models.RepoStateError, 81 81 + ); err != nil { 82 82 + return fmt.Errorf("marking repo desynchronized: %w", err) 83 83 + } 84 84 + return nil 85 85 + } 86 86 + 87 87 + func FinishResync(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey, state models.RepoState, errMsg string, retryCount int, retryAfter int64) error { 88 88 + if _, err := e.ExecContext(ctx, 89 89 + `update repos 90 90 + set error_msg = $1, 91 91 + retry_count = $2, 92 92 + retry_after = $3, 93 93 + state = case when state = $4 then $5 else state end 94 94 + where did = $6 and rkey = $7`, 95 95 + errMsg, 96 96 + retryCount, 97 97 + retryAfter, 98 98 + models.RepoStateResyncing, 99 99 + state, 100 100 + did, rkey, 101 101 + ); err != nil { 102 102 + return fmt.Errorf("finishing resync: %w", err) 103 103 + } 104 104 + return nil 105 105 + } 106 106 + 70 107 func DeleteRepo(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey) error { 71 108 if _, err := e.ExecContext(ctx, 72 109 `delete from repos where did = $1 and rkey = $2`,

+3 -9

knotmirror/knotstream/slurper.go

reviewed

··· 312 312 } 313 313 l = l.With("repoAt", curr.AtUri()) 314 314 315 315 - // TODO: should plan resync to resyncBuffer on RepoStateResyncing 316 316 - if curr.State != models.RepoStateActive { 317 317 - l.Debug("skipping non-active repo") 315 315 + if curr.State == models.RepoStateSuspended || curr.State == models.RepoStatePending { 316 316 + l.Debug("skipping repo", "state", curr.State) 318 317 knotstreamEventsSkipped.Inc() 319 318 return nil 320 319 } ··· 325 324 return nil 326 325 } 327 326 328 328 - // if curr.State == models.RepoStateResyncing { 329 329 - // firehoseEventsSkipped.Inc() 330 330 - // return fp.events.addToResyncBuffer(ctx, commit) 331 331 - // } 332 332 - 333 327 // can't skip anything, update repo state 334 334 - if err := db.UpdateRepoState(ctx, s.db, curr.Did, curr.Rkey, models.RepoStateDesynchronized); err != nil { 328 328 + if err := db.MarkDesynchronized(ctx, s.db, curr.Did, curr.Rkey); err != nil { 335 329 return err 336 330 } 337 331

+80 -72

knotmirror/resyncer.go

reviewed

··· 57 57 } 58 58 59 59 func (r *Resyncer) Start(ctx context.Context) { 60 60 - for i := 0; i < r.parallelism; i++ { 60 60 + for i := range r.parallelism { 61 61 go r.runResyncWorker(ctx, i) 62 62 } 63 63 } ··· 71 71 return 72 72 default: 73 73 } 74 74 - repoAt, found, err := r.claimResyncJob(ctx) 74 74 + repoAt, jobCtx, found, err := r.claimResyncJob(ctx) 75 75 if err != nil { 76 76 l.Error("failed to claim resync job", "error", err) 77 77 time.Sleep(time.Second) ··· 82 82 continue 83 83 } 84 84 l.Info("processing resync", "aturi", repoAt) 85 85 - if err := r.resyncRepo(ctx, repoAt); err != nil { 85 85 + if err := r.resyncRepo(ctx, jobCtx, repoAt); err != nil { 86 86 l.Error("resync failed", "aturi", repoAt, "error", err) 87 87 } 88 88 } 89 89 } 90 90 91 91 - func (r *Resyncer) registerRunning(repo syntax.ATURI, cancel context.CancelFunc) { 91 91 + func (r *Resyncer) finalizeJob(repo syntax.ATURI) { 92 92 r.runningJobsMu.Lock() 93 93 defer r.runningJobsMu.Unlock() 94 94 95 95 - if _, exists := r.runningJobs[repo]; exists { 96 96 - return 95 95 + if cancel, ok := r.runningJobs[repo]; ok { 96 96 + cancel() 97 97 + delete(r.runningJobs, repo) 97 98 } 98 98 - r.runningJobs[repo] = cancel 99 99 - } 100 100 - 101 101 - func (r *Resyncer) unregisterRunning(repo syntax.ATURI) { 102 102 - r.runningJobsMu.Lock() 103 103 - defer r.runningJobsMu.Unlock() 104 104 - 105 105 - delete(r.runningJobs, repo) 106 99 } 107 100 108 101 func (r *Resyncer) CancelResyncJob(repo syntax.ATURI) { 109 109 - r.runningJobsMu.Lock() 110 110 - defer r.runningJobsMu.Unlock() 111 111 - 112 112 - cancel, ok := r.runningJobs[repo] 113 113 - if !ok { 114 114 - return 115 115 - } 116 116 - delete(r.runningJobs, repo) 117 117 - cancel() 102 102 + r.finalizeJob(repo) 118 103 } 119 104 120 105 // TriggerResyncJob manually triggers the resync job 121 106 func (r *Resyncer) TriggerResyncJob(ctx context.Context, repoAt syntax.ATURI) error { 107 107 + res, err := r.db.ExecContext(ctx, 108 108 + `update repos 109 109 + set state = $1, retry_after = $2 110 110 + where at_uri = $3 and state not in ($4, $5)`, 111 111 + models.RepoStatePending, 112 112 + int64(-1), 113 113 + repoAt, 114 114 + models.RepoStateResyncing, 115 115 + models.RepoStateSuspended, 116 116 + ) 117 117 + if err != nil { 118 118 + return fmt.Errorf("triggering resync: %w", err) 119 119 + } 120 120 + n, err := res.RowsAffected() 121 121 + if err != nil { 122 122 + return fmt.Errorf("triggering resync: %w", err) 123 123 + } 124 124 + if n > 0 { 125 125 + return nil 126 126 + } 127 127 + 122 128 repo, err := db.GetRepoByAtUri(ctx, r.db, repoAt) 123 129 if err != nil { 124 130 return fmt.Errorf("failed to get repo: %w", err) ··· 126 132 if repo == nil { 127 133 return fmt.Errorf("repo not found: %s", repoAt) 128 134 } 129 129 - 130 130 - if repo.State == models.RepoStateResyncing { 131 131 - return fmt.Errorf("repo already resyncing") 132 132 - } 133 133 - 134 134 - repo.State = models.RepoStatePending 135 135 - repo.RetryAfter = -1 // resyncer will prioritize this 136 136 - 137 137 - if err := db.UpsertRepo(ctx, r.db, repo); err != nil { 138 138 - return fmt.Errorf("updating repo state to pending %w", err) 139 139 - } 140 140 - return nil 135 135 + return fmt.Errorf("cannot trigger resync: repo in state %s", repo.State) 141 136 } 142 137 143 143 - func (r *Resyncer) claimResyncJob(ctx context.Context) (syntax.ATURI, bool, error) { 138 138 + func (r *Resyncer) claimResyncJob(ctx context.Context) (syntax.ATURI, context.Context, bool, error) { 144 139 // use mutex to prevent duplicated jobs 145 140 r.claimJobMu.Lock() 146 141 defer r.claimJobMu.Unlock() 147 142 148 148 - var repoAt syntax.ATURI 149 149 - now := time.Now().Unix() 150 150 - if err := r.db.QueryRowContext(ctx, 151 151 - `update repos 143 143 + r.runningJobsMu.Lock() 144 144 + excludes := make([]any, 0, len(r.runningJobs)) 145 145 + for aturi := range r.runningJobs { 146 146 + excludes = append(excludes, string(aturi)) 147 147 + } 148 148 + r.runningJobsMu.Unlock() 149 149 + 150 150 + args := []any{ 151 151 + models.RepoStateResyncing, 152 152 + models.RepoStatePending, models.RepoStateDesynchronized, models.RepoStateError, 153 153 + time.Now().Unix(), 154 154 + } 155 155 + excludeClause := "" 156 156 + if len(excludes) > 0 { 157 157 + base := len(args) + 1 158 158 + placeholders := make([]string, len(excludes)) 159 159 + for i := range excludes { 160 160 + placeholders[i] = fmt.Sprintf("$%d", base+i) 161 161 + } 162 162 + excludeClause = " and at_uri not in (" + strings.Join(placeholders, ",") + ")" 163 163 + args = append(args, excludes...) 164 164 + } 165 165 + 166 166 + query := `update repos 152 167 set state = $1 153 168 where at_uri = ( 154 169 select at_uri from repos 155 170 where state in ($2, $3, $4) 156 156 - and (retry_after = -1 or retry_after = 0 or retry_after < $5) 171 171 + and (retry_after = -1 or retry_after = 0 or retry_after < $5)` + excludeClause + ` 157 172 order by 158 173 (retry_after = -1) desc, 159 174 (retry_after = 0) desc, 160 175 retry_after 161 176 limit 1 162 177 ) 163 163 - returning at_uri 164 164 - `, 165 165 - models.RepoStateResyncing, 166 166 - models.RepoStatePending, models.RepoStateDesynchronized, models.RepoStateError, 167 167 - now, 168 168 - ).Scan(&repoAt); err != nil { 178 178 + returning at_uri` 179 179 + 180 180 + var repoAt syntax.ATURI 181 181 + if err := r.db.QueryRowContext(ctx, query, args...).Scan(&repoAt); err != nil { 169 182 if errors.Is(err, sql.ErrNoRows) { 170 170 - return "", false, nil 183 183 + return "", nil, false, nil 171 184 } 172 172 - return "", false, err 185 185 + return "", nil, false, err 173 186 } 174 187 175 175 - return repoAt, true, nil 188 188 + jobCtx, cancel := context.WithCancel(ctx) 189 189 + r.runningJobsMu.Lock() 190 190 + r.runningJobs[repoAt] = cancel 191 191 + r.runningJobsMu.Unlock() 192 192 + 193 193 + return repoAt, jobCtx, true, nil 176 194 } 177 195 178 178 - func (r *Resyncer) resyncRepo(ctx context.Context, repoAt syntax.ATURI) error { 196 196 + func (r *Resyncer) resyncRepo(ctx, jobCtx context.Context, repoAt syntax.ATURI) error { 179 197 // ctx, span := tracer.Start(ctx, "resyncRepo") 180 198 // span.SetAttributes(attribute.String("aturi", repoAt)) 181 199 // defer span.End() ··· 183 201 resyncsStarted.Inc() 184 202 startTime := time.Now() 185 203 186 186 - jobCtx, cancel := context.WithCancel(ctx) 187 187 - r.registerRunning(repoAt, cancel) 188 188 - defer r.unregisterRunning(repoAt) 204 204 + defer r.finalizeJob(repoAt) 189 205 190 190 - success, err := r.doResync(jobCtx, repoAt) 206 206 + success, err := r.doResync(ctx, jobCtx, repoAt) 191 207 if !success { 192 208 resyncsFailed.Inc() 193 209 resyncDuration.Observe(time.Since(startTime).Seconds()) ··· 199 215 return nil 200 216 } 201 217 202 202 - func (r *Resyncer) doResync(ctx context.Context, repoAt syntax.ATURI) (bool, error) { 218 218 + func (r *Resyncer) doResync(ctx, jobCtx context.Context, repoAt syntax.ATURI) (bool, error) { 203 219 // ctx, span := tracer.Start(ctx, "doResync") 204 220 // span.SetAttributes(attribute.String("aturi", repoAt)) 205 221 // defer span.End() 206 222 207 207 - repo, err := db.GetRepoByAtUri(ctx, r.db, repoAt) 223 223 + repo, err := db.GetRepoByAtUri(jobCtx, r.db, repoAt) 208 224 if err != nil { 209 225 return false, fmt.Errorf("failed to get repo: %w", err) 210 226 } ··· 222 238 // HACK: check knot reachability with short timeout before running actual fetch. 223 239 // This is crucial as git-cli doesn't support http connection timeout. 224 240 // `http.lowSpeedTime` is only applied _after_ the connection. 225 225 - if err := r.checkKnotReachability(ctx, repo); err != nil { 241 241 + if err := r.checkKnotReachability(jobCtx, repo); err != nil { 226 242 if isRateLimitError(err) { 227 243 r.knotBackoffMu.Lock() 228 244 r.knotBackoff[repo.KnotDomain] = time.Now().Add(10 * time.Second) ··· 237 253 if repo.RetryAfter == -1 { 238 254 timeout = r.manualResyncTimeout 239 255 } 240 240 - fetchCtx, cancel := context.WithTimeout(ctx, timeout) 256 256 + fetchCtx, cancel := context.WithTimeout(jobCtx, timeout) 241 257 defer cancel() 242 258 243 259 if err := r.gitm.Sync(fetchCtx, repo); err != nil { ··· 246 262 247 263 // repo.GitRev = <processed git.refUpdate revision> 248 264 // repo.RepoSha = <sha256 sum of git refs> 249 249 - repo.State = models.RepoStateActive 250 250 - repo.ErrorMsg = "" 251 251 - repo.RetryCount = 0 252 252 - repo.RetryAfter = 0 253 253 - if err := db.UpsertRepo(ctx, r.db, repo); err != nil { 265 265 + if err := db.FinishResync(ctx, r.db, repo.Did, repo.Rkey, models.RepoStateActive, "", 0, 0); err != nil { 254 266 return false, fmt.Errorf("updating repo state to active %w", err) 255 267 } 256 268 return true, nil ··· 328 340 errMsg = err.Error() 329 341 } 330 342 331 331 - repo, err := db.GetRepoByAtUri(ctx, r.db, repoAt) 332 332 - if err != nil { 333 333 - return fmt.Errorf("failed to get repo: %w", err) 343 343 + repo, getErr := db.GetRepoByAtUri(ctx, r.db, repoAt) 344 344 + if getErr != nil { 345 345 + return fmt.Errorf("failed to get repo: %w", getErr) 334 346 } 335 347 if repo == nil { 336 348 return fmt.Errorf("failed to get repo. repo '%s' doesn't exist in db", repoAt) ··· 343 355 // remove null bytes 344 356 errMsg = strings.ReplaceAll(errMsg, "\x00", "") 345 357 346 346 - repo.State = state 347 347 - repo.ErrorMsg = errMsg 348 348 - repo.RetryCount = retryCount 349 349 - repo.RetryAfter = retryAfter 350 350 - if err := db.UpsertRepo(ctx, r.db, repo); err != nil { 358 358 + if err := db.FinishResync(ctx, r.db, repo.Did, repo.Rkey, state, errMsg, retryCount, retryAfter); err != nil { 351 359 return fmt.Errorf("failed to update repo state: %w", err) 352 360 } 353 361 return nil

+225

knotmirror/resyncer_test.go

reviewed

··· 1 1 + package knotmirror_test 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "database/sql" 6 6 + "io" 7 7 + "log/slog" 8 8 + "net/http" 9 9 + "net/http/httptest" 10 10 + "os" 11 11 + "sync" 12 12 + "testing" 13 13 + "time" 14 14 + 15 15 + "github.com/bluesky-social/indigo/atproto/syntax" 16 16 + "github.com/stretchr/testify/require" 17 17 + "github.com/testcontainers/testcontainers-go" 18 18 + "github.com/testcontainers/testcontainers-go/modules/postgres" 19 19 + "github.com/testcontainers/testcontainers-go/wait" 20 20 + 21 21 + "tangled.org/core/api/tangled" 22 22 + "tangled.org/core/knotmirror" 23 23 + "tangled.org/core/knotmirror/config" 24 24 + "tangled.org/core/knotmirror/db" 25 25 + "tangled.org/core/knotmirror/knotstream" 26 26 + "tangled.org/core/knotmirror/models" 27 27 + ) 28 28 + 29 29 + type blockingGitm struct { 30 30 + mu sync.Mutex 31 31 + calls []syntax.ATURI 32 32 + started chan struct{} 33 33 + release chan struct{} 34 34 + } 35 35 + 36 36 + func newBlockingGitm() *blockingGitm { 37 37 + return &blockingGitm{ 38 38 + started: make(chan struct{}, 16), 39 39 + release: make(chan struct{}, 16), 40 40 + } 41 41 + } 42 42 + 43 43 + func (f *blockingGitm) Exist(repo *models.Repo) (bool, error) { return true, nil } 44 44 + func (f *blockingGitm) RemoteSetUrl(ctx context.Context, repo *models.Repo) error { return nil } 45 45 + func (f *blockingGitm) Clone(ctx context.Context, repo *models.Repo) error { return nil } 46 46 + func (f *blockingGitm) Fetch(ctx context.Context, repo *models.Repo) error { return nil } 47 47 + 48 48 + func (f *blockingGitm) Sync(ctx context.Context, repo *models.Repo) error { 49 49 + f.mu.Lock() 50 50 + f.calls = append(f.calls, repo.AtUri()) 51 51 + f.mu.Unlock() 52 52 + select { 53 53 + case f.started <- struct{}{}: 54 54 + case <-ctx.Done(): 55 55 + return ctx.Err() 56 56 + } 57 57 + select { 58 58 + case <-f.release: 59 59 + return nil 60 60 + case <-ctx.Done(): 61 61 + return ctx.Err() 62 62 + } 63 63 + } 64 64 + 65 65 + func (f *blockingGitm) callCount() int { 66 66 + f.mu.Lock() 67 67 + defer f.mu.Unlock() 68 68 + return len(f.calls) 69 69 + } 70 70 + 71 71 + var _ knotmirror.GitMirrorManager = (*blockingGitm)(nil) 72 72 + 73 73 + func startPostgres(t *testing.T, ctx context.Context) *sql.DB { 74 74 + t.Helper() 75 75 + 76 76 + if url := os.Getenv("TEST_POSTGRES_URL"); url != "" { 77 77 + database, err := db.Make(ctx, url, 4) 78 78 + require.NoError(t, err) 79 79 + _, err = database.ExecContext(ctx, `truncate table repos, hosts`) 80 80 + require.NoError(t, err) 81 81 + t.Cleanup(func() { 82 82 + _ = database.Close() 83 83 + }) 84 84 + return database 85 85 + } 86 86 + 87 87 + pg, err := postgres.Run(ctx, "docker.io/library/postgres:16-alpine", 88 88 + postgres.WithDatabase("mirror"), 89 89 + postgres.WithUsername("tnglr"), 90 90 + postgres.WithPassword("test"), 91 91 + testcontainers.WithWaitStrategy( 92 92 + wait.ForLog("database system is ready to accept connections"). 93 93 + WithOccurrence(2). 94 94 + WithStartupTimeout(60*time.Second), 95 95 + ), 96 96 + ) 97 97 + if err != nil { 98 98 + t.Skipf("postgres container unavailable, is podman/docker running: %v", err) 99 99 + } 100 100 + t.Cleanup(func() { 101 101 + _ = pg.Terminate(context.Background()) 102 102 + }) 103 103 + 104 104 + url, err := pg.ConnectionString(ctx, "sslmode=disable") 105 105 + require.NoError(t, err) 106 106 + 107 107 + database, err := db.Make(ctx, url, 4) 108 108 + require.NoError(t, err) 109 109 + t.Cleanup(func() { 110 110 + _ = database.Close() 111 111 + }) 112 112 + return database 113 113 + } 114 114 + 115 115 + func testLogger() *slog.Logger { 116 116 + return slog.New(slog.NewTextHandler(io.Discard, nil)) 117 117 + } 118 118 + 119 119 + func mockKnot(t *testing.T) *httptest.Server { 120 120 + t.Helper() 121 121 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 122 122 + w.Header().Set("Content-Type", "application/x-git-upload-pack-advertisement") 123 123 + w.WriteHeader(http.StatusOK) 124 124 + _, _ = w.Write([]byte("001e# service=git-upload-pack\n")) 125 125 + })) 126 126 + t.Cleanup(srv.Close) 127 127 + return srv 128 128 + } 129 129 + 130 130 + func seedActiveRepo(t *testing.T, ctx context.Context, database *sql.DB, knotURL string) *models.Repo { 131 131 + t.Helper() 132 132 + repo := &models.Repo{ 133 133 + Did: syntax.DID("did:plc:testingtestingtestingtest"), 134 134 + Rkey: syntax.RecordKey("3kaaaaaaaaaaaa"), 135 135 + Name: "race-repo", 136 136 + KnotDomain: knotURL, 137 137 + State: models.RepoStateActive, 138 138 + } 139 139 + require.NoError(t, db.UpsertRepo(ctx, database, repo)) 140 140 + return repo 141 141 + } 142 142 + 143 143 + func mkEvent(rkey string, oldSha, newSha string, repo *models.Repo) *knotstream.LegacyGitEvent { 144 144 + owner := repo.Did.String() 145 145 + repoDid := repo.Did.String() 146 146 + return &knotstream.LegacyGitEvent{ 147 147 + Rkey: rkey, 148 148 + Nsid: tangled.GitRefUpdateNSID, 149 149 + Event: tangled.GitRefUpdate{ 150 150 + OwnerDid: &owner, 151 151 + RepoName: repo.Name, 152 152 + RepoDid: &repoDid, 153 153 + OldSha: oldSha, 154 154 + NewSha: newSha, 155 155 + Ref: "refs/heads/main", 156 156 + }, 157 157 + } 158 158 + } 159 159 + 160 160 + func TestEventDuringResyncTriggersSecondSync(t *testing.T) { 161 161 + ctx, cancel := context.WithCancel(context.Background()) 162 162 + defer cancel() 163 163 + 164 164 + database := startPostgres(t, ctx) 165 165 + knotSrv := mockKnot(t) 166 166 + repo := seedActiveRepo(t, ctx, database, knotSrv.URL) 167 167 + 168 168 + gitm := newBlockingGitm() 169 169 + logger := testLogger() 170 170 + cfg := &config.Config{ 171 171 + KnotUseSSL: false, 172 172 + GitRepoFetchTimeout: 10 * time.Second, 173 173 + ResyncParallelism: 1, 174 174 + } 175 175 + 176 176 + slurper := knotstream.NewKnotSlurper(logger, database, cfg.Slurper) 177 177 + resyncer := knotmirror.NewResyncer(logger, database, gitm, cfg) 178 178 + resyncer.Start(ctx) 179 179 + 180 180 + eventA := mkEvent("3evtaaaaaaaaaa", "0000000000000000000000000000000000000000", "1111111111111111111111111111111111111111", repo) 181 181 + require.NoError(t, slurper.ProcessLegacyGitRefUpdate(ctx, "testsrc", eventA)) 182 182 + 183 183 + got, err := db.GetRepoByAtUri(ctx, database, repo.AtUri()) 184 184 + require.NoError(t, err) 185 185 + require.Equal(t, models.RepoStateDesynchronized, got.State, "event A should mark repo desynchronized") 186 186 + 187 187 + select { 188 188 + case <-gitm.started: 189 189 + case <-time.After(10 * time.Second): 190 190 + t.Fatal("timeout waiting for first Sync to start") 191 191 + } 192 192 + 193 193 + got, err = db.GetRepoByAtUri(ctx, database, repo.AtUri()) 194 194 + require.NoError(t, err) 195 195 + require.Equal(t, models.RepoStateResyncing, got.State, "state should be resyncing while fetch is in flight") 196 196 + 197 197 + eventB := mkEvent("3evtbbbbbbbbbb", "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", repo) 198 198 + require.NoError(t, slurper.ProcessLegacyGitRefUpdate(ctx, "testsrc", eventB)) 199 199 + 200 200 + gitm.release <- struct{}{} 201 201 + 202 202 + secondSyncStarted := false 203 203 + select { 204 204 + case <-gitm.started: 205 205 + secondSyncStarted = true 206 206 + gitm.release <- struct{}{} 207 207 + case <-time.After(10 * time.Second): 208 208 + } 209 209 + 210 210 + deadline := time.Now().Add(10 * time.Second) 211 211 + var finalState models.RepoState 212 212 + for time.Now().Before(deadline) { 213 213 + got, err = db.GetRepoByAtUri(ctx, database, repo.AtUri()) 214 214 + require.NoError(t, err) 215 215 + finalState = got.State 216 216 + if finalState == models.RepoStateActive { 217 217 + break 218 218 + } 219 219 + time.Sleep(50 * time.Millisecond) 220 220 + } 221 221 + 222 222 + require.Truef(t, secondSyncStarted, "event B arriving during resync should trigger a second Sync, got %d total Sync calls", gitm.callCount()) 223 223 + require.GreaterOrEqual(t, gitm.callCount(), 2, "expected at least 2 Sync calls") 224 224 + require.Equal(t, models.RepoStateActive, finalState, "repo should end in active state") 225 225 + }