forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
knotmirror: add `knotBackoff` and reachability test
git-cli doesn't support http connection timeout, so we cannot set short
30s connection timeout on git fetch. We don't want to put operation
timeout that short because intial `git clone` can take pretty long.
go-git does expose http client but only globally and is less efficient
than cli. So as a hack, just fetch remote server to check if knot is
available and is valid git remote server
Signed-off-by: Seongmin Lee <git@boltless.me>
authored by
Seongmin Lee
and committed by
tangled.org
51817c4c
a3a409ec
+85
-2
+85
-2
knotmirror/resyncer.go
+85
-2
knotmirror/resyncer.go
···
7
7
"fmt"
8
8
"log/slog"
9
9
"math/rand"
10
+
"net/http"
11
+
"net/url"
10
12
"strings"
11
13
"sync"
12
14
"time"
···
33
31
repoFetchTimeout time.Duration
34
32
manualResyncTimeout time.Duration
35
33
parallelism int
34
+
35
+
knotBackoff map[string]time.Time
36
+
knotBackoffMu sync.RWMutex
36
37
}
37
38
38
39
func NewResyncer(l *slog.Logger, db *sql.DB, gitm GitMirrorManager, cfg *config.Config) *Resyncer {
···
49
44
repoFetchTimeout: cfg.GitRepoFetchTimeout,
50
45
manualResyncTimeout: 30 * time.Minute,
51
46
parallelism: cfg.ResyncParallelism,
47
+
48
+
knotBackoff: make(map[string]time.Time),
52
49
}
53
50
}
54
51
···
210
203
return false, nil
211
204
}
212
205
213
-
// TODO: check if Knot is on backoff list. If so, return (false, nil)
214
-
// TODO: detect rate limit error (http.StatusTooManyRequests) to put Knot in backoff list
206
+
r.knotBackoffMu.RLock()
207
+
backoffUntil, inBackoff := r.knotBackoff[repo.KnotDomain]
208
+
r.knotBackoffMu.RUnlock()
209
+
if inBackoff && time.Now().Before(backoffUntil) {
210
+
return false, nil
211
+
}
212
+
213
+
// HACK: check knot reachability with short timeout before running actual fetch.
214
+
// This is crucial as git-cli doesn't support http connection timeout.
215
+
// `http.lowSpeedTime` is only applied _after_ the connection.
216
+
if err := r.checkKnotReachability(ctx, repo); err != nil {
217
+
if isRateLimitError(err) {
218
+
r.knotBackoffMu.Lock()
219
+
r.knotBackoff[repo.KnotDomain] = time.Now().Add(10 * time.Second)
220
+
r.knotBackoffMu.Unlock()
221
+
return false, nil
222
+
}
223
+
// TODO: suspend repo on 404. KnotStream updates will change the repo state back online
224
+
return false, fmt.Errorf("knot unreachable: %w", err)
225
+
}
215
226
216
227
timeout := r.repoFetchTimeout
217
228
if repo.RetryAfter == -1 {
···
252
227
return false, fmt.Errorf("updating repo state to active %w", err)
253
228
}
254
229
return true, nil
230
+
}
231
+
232
+
type knotStatusError struct {
233
+
StatusCode int
234
+
}
235
+
236
+
func (ke *knotStatusError) Error() string {
237
+
return fmt.Sprintf("request failed with status code (HTTP %d)", ke.StatusCode)
238
+
}
239
+
240
+
func isRateLimitError(err error) bool {
241
+
var knotErr *knotStatusError
242
+
if errors.As(err, &knotErr) {
243
+
return knotErr.StatusCode == http.StatusTooManyRequests
244
+
}
245
+
return false
246
+
}
247
+
248
+
// checkKnotReachability checks if Knot is reachable and is valid git remote server
249
+
func (r *Resyncer) checkKnotReachability(ctx context.Context, repo *models.Repo) error {
250
+
repoUrl, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), true)
251
+
if err != nil {
252
+
return err
253
+
}
254
+
255
+
repoUrl += "/info/refs?service=git-upload-pack"
256
+
257
+
client := http.Client{
258
+
Timeout: 30 * time.Second,
259
+
}
260
+
req, err := http.NewRequestWithContext(ctx, "GET", repoUrl, nil)
261
+
if err != nil {
262
+
return err
263
+
}
264
+
req.Header.Set("User-Agent", "git/2.x")
265
+
req.Header.Set("Accept", "*/*")
266
+
267
+
resp, err := client.Do(req)
268
+
if err != nil {
269
+
var uerr *url.Error
270
+
if errors.As(err, &uerr) {
271
+
return fmt.Errorf("request failed: %w", uerr.Unwrap())
272
+
}
273
+
return fmt.Errorf("request failed: %w", err)
274
+
}
275
+
defer resp.Body.Close()
276
+
277
+
if resp.StatusCode != http.StatusOK {
278
+
return &knotStatusError{resp.StatusCode}
279
+
}
280
+
281
+
// check if target is git server
282
+
ct := resp.Header.Get("Content-Type")
283
+
if !strings.Contains(ct, "application/x-git-upload-pack-advertisement") {
284
+
return fmt.Errorf("unexpected content-type: %s", ct)
285
+
}
286
+
287
+
return nil
255
288
}
256
289
257
290
func (r *Resyncer) handleResyncFailure(ctx context.Context, repoAt syntax.ATURI, err error) error {