alice.mosphere.at / indigo
this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

Some aditional rate limit controls at per-PDS levels to help mitigate… (#572)

… network abuse

authored by

Jaz and committed by
GitHub d2be3e13 47edba75

+1162 -335
+4
Makefile
··· 73 73 .env: 74 74 if [ ! -f ".env" ]; then cp example.dev.env .env; fi 75 75 76 + .PHONY: run-postgres 77 + run-postgres: .env ## Runs a local postgres instance 78 + docker compose -f cmd/bigsky/docker-compose.yml up -d 79 + 76 80 .PHONY: run-dev-bgs 77 81 run-dev-bgs: .env ## Runs 'bigsky' BGS for local dev 78 82 GOLOG_LOG_LEVEL=info go run ./cmd/bigsky --admin-key localdev
+29 -7
api/extra.go
··· 76 76 } 77 77 78 78 type ProdHandleResolver struct { 79 + client *http.Client 80 + resolver *net.Resolver 79 81 ReqMod func(*http.Request, string) error 80 82 FailCache *arc.ARCCache[string, *failCacheItem] 81 83 } 82 84 83 - func NewProdHandleResolver(failureCacheSize int) (*ProdHandleResolver, error) { 85 + func NewProdHandleResolver(failureCacheSize int, resolveAddr string, forceUDP bool) (*ProdHandleResolver, error) { 84 86 failureCache, err := arc.NewARC[string, *failCacheItem](failureCacheSize) 85 87 if err != nil { 86 88 return nil, err 87 89 } 88 90 91 + if resolveAddr == "" { 92 + resolveAddr = "1.1.1.1:53" 93 + } 94 + 95 + c := http.Client{ 96 + Transport: otelhttp.NewTransport(http.DefaultTransport), 97 + Timeout: time.Second * 10, 98 + } 99 + 100 + r := &net.Resolver{ 101 + PreferGo: true, 102 + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { 103 + d := net.Dialer{ 104 + Timeout: time.Second * 10, 105 + } 106 + if forceUDP { 107 + network = "udp" 108 + } 109 + return d.DialContext(ctx, network, resolveAddr) 110 + }, 111 + } 112 + 89 113 return &ProdHandleResolver{ 90 114 FailCache: failureCache, 115 + client: &c, 116 + resolver: r, 91 117 }, nil 92 118 } 93 119 ··· 166 192 } 167 193 168 194 func (dr *ProdHandleResolver) resolveWellKnown(ctx context.Context, handle string) (string, error) { 169 - c := http.Client{ 170 - Transport: otelhttp.NewTransport(http.DefaultTransport), 171 - } 172 - 173 195 req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/.well-known/atproto-did", handle), nil) 174 196 if err != nil { 175 197 return "", err ··· 183 205 184 206 req = req.WithContext(ctx) 185 207 186 - resp, err := c.Do(req) 208 + resp, err := dr.client.Do(req) 187 209 if err != nil { 188 210 return "", fmt.Errorf("failed to resolve handle (%s) through HTTP well-known route: %s", handle, err) 189 211 } ··· 209 231 } 210 232 211 233 func (dr *ProdHandleResolver) resolveDNS(ctx context.Context, handle string) (string, error) { 212 - res, err := net.LookupTXT("_atproto." + handle) 234 + res, err := dr.resolver.LookupTXT(ctx, "_atproto."+handle) 213 235 if err != nil { 214 236 return "", fmt.Errorf("handle lookup failed: %w", err) 215 237 }
+114 -89
bgs/admin.go
··· 5 5 "errors" 6 6 "fmt" 7 7 "net/http" 8 + "net/url" 9 + "slices" 8 10 "strconv" 9 11 "strings" 10 12 "time" ··· 92 94 } 93 95 94 96 type rateLimit struct { 95 - MaxEventsPerSecond float64 `json:"MaxEventsPerSecond"` 96 - TokenCount float64 `json:"TokenCount"` 97 + Max float64 `json:"Max"` 98 + WindowSeconds float64 `json:"Window"` 97 99 } 98 100 99 101 type enrichedPDS struct { 100 102 models.PDS 101 103 HasActiveConnection bool `json:"HasActiveConnection"` 102 104 EventsSeenSinceStartup uint64 `json:"EventsSeenSinceStartup"` 103 - IngestRate rateLimit `json:"IngestRate"` 105 + PerSecondEventRate rateLimit `json:"PerSecondEventRate"` 106 + PerHourEventRate rateLimit `json:"PerHourEventRate"` 107 + PerDayEventRate rateLimit `json:"PerDayEventRate"` 104 108 CrawlRate rateLimit `json:"CrawlRate"` 105 109 UserCount int64 `json:"UserCount"` 106 110 } ··· 120 124 121 125 activePDSHosts := bgs.slurper.GetActiveList() 122 126 123 - var userCounts []UserCount 124 - if err := bgs.db.Model(&User{}). 125 - Select("pds, count(*) as user_count"). 126 - Group("pds"). 127 - Find(&userCounts).Error; err != nil { 128 - return err 129 - } 130 - 131 - // Create a map for fast lookup 132 - userCountMap := make(map[uint]int64) 133 - for _, count := range userCounts { 134 - userCountMap[count.PDSID] = count.UserCount 135 - } 136 - 137 127 for i, p := range pds { 138 128 enrichedPDSs[i].PDS = p 139 129 enrichedPDSs[i].HasActiveConnection = false ··· 149 139 continue 150 140 } 151 141 enrichedPDSs[i].EventsSeenSinceStartup = uint64(m.Counter.GetValue()) 152 - enrichedPDSs[i].UserCount = userCountMap[p.ID] 153 142 154 - // Get the ingest rate limit for this PDS 155 - ingestRate := rateLimit{ 156 - MaxEventsPerSecond: p.RateLimit, 143 + enrichedPDSs[i].PerSecondEventRate = rateLimit{ 144 + Max: p.RateLimit, 145 + WindowSeconds: 1, 157 146 } 158 147 159 - limiter := bgs.slurper.GetLimiter(p.ID) 160 - if limiter != nil { 161 - ingestRate.TokenCount = limiter.Tokens() 148 + enrichedPDSs[i].PerHourEventRate = rateLimit{ 149 + Max: float64(p.HourlyEventLimit), 150 + WindowSeconds: 3600, 162 151 } 163 152 164 - enrichedPDSs[i].IngestRate = ingestRate 153 + enrichedPDSs[i].PerDayEventRate = rateLimit{ 154 + Max: float64(p.DailyEventLimit), 155 + WindowSeconds: 86400, 156 + } 165 157 166 158 // Get the crawl rate limit for this PDS 167 159 crawlRate := rateLimit{ 168 - MaxEventsPerSecond: p.CrawlRateLimit, 169 - } 170 - 171 - limiter = bgs.repoFetcher.GetLimiter(p.ID) 172 - if limiter != nil { 173 - crawlRate.TokenCount = limiter.Tokens() 160 + Max: p.CrawlRateLimit, 161 + WindowSeconds: 1, 174 162 } 175 163 176 164 enrichedPDSs[i].CrawlRate = crawlRate ··· 338 326 }) 339 327 } 340 328 341 - func (bgs *BGS) handleAdminChangePDSRateLimit(e echo.Context) error { 342 - host := strings.TrimSpace(e.QueryParam("host")) 343 - if host == "" { 344 - return &echo.HTTPError{ 345 - Code: 400, 346 - Message: "must pass a valid host", 347 - } 348 - } 329 + type RateLimitChangeRequest struct { 330 + Host string `json:"host"` 331 + PerSecond int64 `json:"per_second"` 332 + PerHour int64 `json:"per_hour"` 333 + PerDay int64 `json:"per_day"` 334 + CrawlRate int64 `json:"crawl_rate"` 335 + RepoLimit int64 `json:"repo_limit"` 336 + } 349 337 350 - // Get the new rate limit 351 - limit, err := strconv.ParseFloat(e.QueryParam("limit"), 64) 352 - if err != nil { 353 - return &echo.HTTPError{ 354 - Code: 400, 355 - Message: "must pass a valid limit", 356 - } 338 + func (bgs *BGS) handleAdminChangePDSRateLimits(e echo.Context) error { 339 + var body RateLimitChangeRequest 340 + if err := e.Bind(&body); err != nil { 341 + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid body: %s", err)) 357 342 } 358 343 359 344 // Get the PDS from the DB 360 345 var pds models.PDS 361 - if err := bgs.db.Where("host = ?", host).First(&pds).Error; err != nil { 346 + if err := bgs.db.Where("host = ?", body.Host).First(&pds).Error; err != nil { 362 347 return err 363 348 } 364 349 365 - // Update the rate limit in the DB 366 - if err := bgs.db.Model(&pds).Update("rate_limit", limit).Error; err != nil { 367 - return err 350 + // Update the rate limits in the DB 351 + pds.RateLimit = float64(body.PerSecond) 352 + pds.HourlyEventLimit = body.PerHour 353 + pds.DailyEventLimit = body.PerDay 354 + pds.CrawlRateLimit = float64(body.CrawlRate) 355 + pds.RepoLimit = body.RepoLimit 356 + 357 + if err := bgs.db.Save(&pds).Error; err != nil { 358 + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Errorf("failed to save rate limit changes: %w", err)) 368 359 } 369 360 370 361 // Update the rate limit in the limiter 371 - limiter := bgs.slurper.GetOrCreateLimiter(pds.ID, limit) 372 - limiter.SetLimit(rate.Limit(limit)) 362 + limits := bgs.slurper.GetOrCreateLimiters(pds.ID, body.PerSecond, body.PerHour, body.PerDay) 363 + limits.PerSecond.SetLimit(body.PerSecond) 364 + limits.PerHour.SetLimit(body.PerHour) 365 + limits.PerDay.SetLimit(body.PerDay) 373 366 374 - return e.JSON(200, map[string]any{ 375 - "success": "true", 376 - }) 377 - } 378 - 379 - func (bgs *BGS) handleAdminChangePDSCrawlLimit(e echo.Context) error { 380 - host := strings.TrimSpace(e.QueryParam("host")) 381 - if host == "" { 382 - return &echo.HTTPError{ 383 - Code: 400, 384 - Message: "must pass a valid host", 385 - } 386 - } 387 - 388 - // Get the new crawl limit 389 - limit, err := strconv.ParseFloat(e.QueryParam("limit"), 64) 390 - if err != nil { 391 - return &echo.HTTPError{ 392 - Code: 400, 393 - Message: "must pass a valid limit", 394 - } 395 - } 396 - 397 - // Get the PDS from the DB 398 - var pds models.PDS 399 - if err := bgs.db.Where("host = ?", host).First(&pds).Error; err != nil { 400 - return err 401 - } 402 - 403 - // Update the crawl limit in the DB 404 - if err := bgs.db.Model(&pds).Update("crawl_rate_limit", limit).Error; err != nil { 405 - return err 406 - } 407 - 408 - // Update the crawl limit in the limiter 409 - limiter := bgs.repoFetcher.GetOrCreateLimiter(pds.ID, limit) 410 - limiter.SetLimit(rate.Limit(limit)) 367 + // Set the crawl rate limit 368 + bgs.repoFetcher.GetOrCreateLimiter(pds.ID, float64(body.CrawlRate)).SetLimit(rate.Limit(body.CrawlRate)) 411 369 412 370 return e.JSON(200, map[string]any{ 413 371 "success": "true", ··· 587 545 return fmt.Errorf("must specify domain in query parameter") 588 546 } 589 547 548 + // Check if the domain is already trusted 549 + trustedDomains := bgs.slurper.GetTrustedDomains() 550 + if slices.Contains(trustedDomains, domain) { 551 + return &echo.HTTPError{ 552 + Code: 400, 553 + Message: "domain is already trusted", 554 + } 555 + } 556 + 590 557 if err := bgs.slurper.AddTrustedDomain(domain); err != nil { 591 558 return err 592 559 } ··· 595 562 "success": true, 596 563 }) 597 564 } 565 + 566 + type AdminRequestCrawlRequest struct { 567 + Hostname string `json:"hostname"` 568 + } 569 + 570 + func (bgs *BGS) handleAdminRequestCrawl(e echo.Context) error { 571 + ctx := e.Request().Context() 572 + 573 + var body AdminRequestCrawlRequest 574 + if err := e.Bind(&body); err != nil { 575 + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid body: %s", err)) 576 + } 577 + 578 + host := body.Hostname 579 + if host == "" { 580 + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname") 581 + } 582 + 583 + if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") { 584 + if bgs.ssl { 585 + host = "https://" + host 586 + } else { 587 + host = "http://" + host 588 + } 589 + } 590 + 591 + u, err := url.Parse(host) 592 + if err != nil { 593 + return echo.NewHTTPError(http.StatusBadRequest, "failed to parse hostname") 594 + } 595 + 596 + if u.Scheme == "http" && bgs.ssl { 597 + return echo.NewHTTPError(http.StatusBadRequest, "this server requires https") 598 + } 599 + 600 + if u.Scheme == "https" && !bgs.ssl { 601 + return echo.NewHTTPError(http.StatusBadRequest, "this server does not support https") 602 + } 603 + 604 + if u.Path != "" { 605 + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without path") 606 + } 607 + 608 + if u.Query().Encode() != "" { 609 + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without query") 610 + } 611 + 612 + host = u.Host // potentially hostname:port 613 + 614 + banned, err := bgs.domainIsBanned(ctx, host) 615 + if banned { 616 + return echo.NewHTTPError(http.StatusUnauthorized, "domain is banned") 617 + } 618 + 619 + // Skip checking if the server is online for now 620 + 621 + return bgs.slurper.SubscribeToPds(ctx, host, true, true) // Override Trusted Domain Check 622 + }
+39 -7
bgs/bgs.go
··· 264 264 e.HideBanner = true 265 265 266 266 e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ 267 - AllowOrigins: []string{"http://localhost:*", "https://bgs.bsky-sandbox.dev"}, 267 + AllowOrigins: []string{"*"}, 268 268 AllowHeaders: []string{echo.HeaderOrigin, echo.HeaderContentType, echo.HeaderAccept, echo.HeaderAuthorization}, 269 269 })) 270 270 ··· 348 348 admin.POST("/repo/verify", bgs.handleAdminVerifyRepo) 349 349 350 350 // PDS-related Admin API 351 + admin.POST("/pds/requestCrawl", bgs.handleAdminRequestCrawl) 351 352 admin.GET("/pds/list", bgs.handleListPDSs) 352 353 admin.POST("/pds/resync", bgs.handleAdminPostResyncPDS) 353 354 admin.GET("/pds/resync", bgs.handleAdminGetResyncPDS) 354 - admin.POST("/pds/changeIngestRateLimit", bgs.handleAdminChangePDSRateLimit) 355 - admin.POST("/pds/changeCrawlRateLimit", bgs.handleAdminChangePDSCrawlLimit) 355 + admin.POST("/pds/changeLimits", bgs.handleAdminChangePDSRateLimits) 356 356 admin.POST("/pds/block", bgs.handleBlockPDS) 357 357 admin.POST("/pds/unblock", bgs.handleUnblockPDS) 358 358 admin.POST("/pds/addTrustedDomain", bgs.handleAdminAddTrustedDomain) ··· 1044 1044 return nil, err 1045 1045 } 1046 1046 1047 - if peering.Blocked { 1048 - return nil, fmt.Errorf("refusing to create user with blocked PDS") 1049 - } 1050 - 1051 1047 ban, err := s.domainIsBanned(ctx, durl.Host) 1052 1048 if err != nil { 1053 1049 return nil, fmt.Errorf("failed to check pds ban status: %w", err) ··· 1074 1070 // TODO: could check other things, a valid response is good enough for now 1075 1071 peering.Host = durl.Host 1076 1072 peering.SSL = (durl.Scheme == "https") 1073 + peering.CrawlRateLimit = float64(s.slurper.DefaultCrawlLimit) 1074 + peering.RepoLimit = s.slurper.DefaultPerSecondLimit 1075 + peering.HourlyEventLimit = s.slurper.DefaultPerHourLimit 1076 + peering.DailyEventLimit = s.slurper.DefaultPerDayLimit 1077 + peering.RepoLimit = s.slurper.DefaultRepoLimit 1077 1078 1078 1079 if s.ssl && !peering.SSL { 1079 1080 return nil, fmt.Errorf("did references non-ssl PDS, this is disallowed in prod: %q %q", did, svc.ServiceEndpoint) ··· 1087 1088 if peering.ID == 0 { 1088 1089 panic("somehow failed to create a pds entry?") 1089 1090 } 1091 + 1092 + if peering.Blocked { 1093 + return nil, fmt.Errorf("refusing to create user with blocked PDS") 1094 + } 1095 + 1096 + if peering.RepoCount >= peering.RepoLimit { 1097 + return nil, fmt.Errorf("refusing to create user on PDS at max repo limit for pds %q", peering.Host) 1098 + } 1099 + 1100 + // Increment the repo count for the PDS 1101 + res := s.db.Model(&models.PDS{}).Where("id = ? AND repo_count < repo_limit", peering.ID).Update("repo_count", gorm.Expr("repo_count + 1")) 1102 + if res.Error != nil { 1103 + return nil, fmt.Errorf("failed to increment repo count for pds %q: %w", peering.Host, res.Error) 1104 + } 1105 + 1106 + if res.RowsAffected == 0 { 1107 + return nil, fmt.Errorf("refusing to create user on PDS at max repo limit for pds %q", peering.Host) 1108 + } 1109 + 1110 + successfullyCreated := false 1111 + 1112 + // Release the count if we fail to create the user 1113 + defer func() { 1114 + if !successfullyCreated { 1115 + if err := s.db.Model(&models.PDS{}).Where("id = ?", peering.ID).Update("repo_count", gorm.Expr("repo_count - 1")).Error; err != nil { 1116 + log.Errorf("failed to decrement repo count for pds: %s", err) 1117 + } 1118 + } 1119 + }() 1090 1120 1091 1121 if len(doc.AlsoKnownAs) == 0 { 1092 1122 return nil, fmt.Errorf("user has no 'known as' field in their DID document") ··· 1211 1241 if err := s.db.Create(subj).Error; err != nil { 1212 1242 return nil, err 1213 1243 } 1244 + 1245 + successfullyCreated = true 1214 1246 1215 1247 return subj, nil 1216 1248 }
+91 -37
bgs/fedmgr.go
··· 9 9 "sync" 10 10 "time" 11 11 12 + "github.com/RussellLuo/slidingwindow" 12 13 comatproto "github.com/bluesky-social/indigo/api/atproto" 13 14 "github.com/bluesky-social/indigo/events" 14 15 "github.com/bluesky-social/indigo/events/schedulers/parallel" ··· 32 33 lk sync.Mutex 33 34 active map[string]*activeSub 34 35 35 - LimitMux sync.RWMutex 36 - Limiters map[uint]*rate.Limiter 37 - DefaultLimit rate.Limit 36 + LimitMux sync.RWMutex 37 + Limiters map[uint]*Limiters 38 + DefaultPerSecondLimit int64 39 + DefaultPerHourLimit int64 40 + DefaultPerDayLimit int64 41 + 38 42 DefaultCrawlLimit rate.Limit 43 + DefaultRepoLimit int64 39 44 40 45 newSubsDisabled bool 41 46 trustedDomains []string ··· 46 51 ssl bool 47 52 } 48 53 54 + type Limiters struct { 55 + PerSecond *slidingwindow.Limiter 56 + PerHour *slidingwindow.Limiter 57 + PerDay *slidingwindow.Limiter 58 + } 59 + 49 60 type SlurperOptions struct { 50 - SSL bool 51 - DefaultIngestLimit rate.Limit 52 - DefaultCrawlLimit rate.Limit 61 + SSL bool 62 + DefaultPerSecondLimit int64 63 + DefaultPerHourLimit int64 64 + DefaultPerDayLimit int64 65 + DefaultCrawlLimit rate.Limit 66 + DefaultRepoLimit int64 53 67 } 54 68 55 69 func DefaultSlurperOptions() *SlurperOptions { 56 70 return &SlurperOptions{ 57 - SSL: false, 58 - DefaultIngestLimit: rate.Limit(50), 59 - DefaultCrawlLimit: rate.Limit(5), 71 + SSL: false, 72 + DefaultPerSecondLimit: 50, 73 + DefaultPerHourLimit: 1500, 74 + DefaultPerDayLimit: 10_000, 75 + DefaultCrawlLimit: rate.Limit(5), 76 + DefaultRepoLimit: 10, 60 77 } 61 78 } 62 79 ··· 73 90 } 74 91 db.AutoMigrate(&SlurpConfig{}) 75 92 s := &Slurper{ 76 - cb: cb, 77 - db: db, 78 - active: make(map[string]*activeSub), 79 - Limiters: make(map[uint]*rate.Limiter), 80 - DefaultLimit: opts.DefaultIngestLimit, 81 - DefaultCrawlLimit: opts.DefaultCrawlLimit, 82 - ssl: opts.SSL, 83 - shutdownChan: make(chan bool), 84 - shutdownResult: make(chan []error), 93 + cb: cb, 94 + db: db, 95 + active: make(map[string]*activeSub), 96 + Limiters: make(map[uint]*Limiters), 97 + DefaultPerSecondLimit: opts.DefaultPerSecondLimit, 98 + DefaultPerHourLimit: opts.DefaultPerHourLimit, 99 + DefaultPerDayLimit: opts.DefaultPerDayLimit, 100 + DefaultCrawlLimit: opts.DefaultCrawlLimit, 101 + DefaultRepoLimit: opts.DefaultRepoLimit, 102 + ssl: opts.SSL, 103 + shutdownChan: make(chan bool), 104 + shutdownResult: make(chan []error), 85 105 } 86 106 if err := s.loadConfig(); err != nil { 87 107 return nil, err ··· 123 143 return s, nil 124 144 } 125 145 126 - func (s *Slurper) GetLimiter(pdsID uint) *rate.Limiter { 146 + func windowFunc() (slidingwindow.Window, slidingwindow.StopFunc) { 147 + return slidingwindow.NewLocalWindow() 148 + } 149 + 150 + func (s *Slurper) GetLimiters(pdsID uint) *Limiters { 127 151 s.LimitMux.RLock() 128 152 defer s.LimitMux.RUnlock() 129 153 return s.Limiters[pdsID] 130 154 } 131 155 132 - func (s *Slurper) GetOrCreateLimiter(pdsID uint, nlimit float64) *rate.Limiter { 156 + func (s *Slurper) GetOrCreateLimiters(pdsID uint, perSecLimit int64, perHourLimit int64, perDayLimit int64) *Limiters { 133 157 s.LimitMux.RLock() 134 158 defer s.LimitMux.RUnlock() 135 159 lim, ok := s.Limiters[pdsID] 136 160 if !ok { 137 - lim = rate.NewLimiter(rate.Limit(nlimit), 1) 161 + perSec, _ := slidingwindow.NewLimiter(time.Second, perSecLimit, windowFunc) 162 + perHour, _ := slidingwindow.NewLimiter(time.Hour, perHourLimit, windowFunc) 163 + perDay, _ := slidingwindow.NewLimiter(time.Hour*24, perDayLimit, windowFunc) 164 + lim = &Limiters{ 165 + PerSecond: perSec, 166 + PerHour: perHour, 167 + PerDay: perDay, 168 + } 138 169 s.Limiters[pdsID] = lim 139 170 } 140 171 141 172 return lim 142 173 } 143 174 144 - func (s *Slurper) SetLimiter(pdsID uint, limiter *rate.Limiter) { 175 + func (s *Slurper) SetLimits(pdsID uint, perSecLimit int64, perHourLimit int64, perDayLimit int64) { 145 176 s.LimitMux.Lock() 146 177 defer s.LimitMux.Unlock() 147 - s.Limiters[pdsID] = limiter 178 + lim, ok := s.Limiters[pdsID] 179 + if !ok { 180 + perSec, _ := slidingwindow.NewLimiter(time.Second, perSecLimit, windowFunc) 181 + perHour, _ := slidingwindow.NewLimiter(time.Hour, perHourLimit, windowFunc) 182 + perDay, _ := slidingwindow.NewLimiter(time.Hour*24, perDayLimit, windowFunc) 183 + lim = &Limiters{ 184 + PerSecond: perSec, 185 + PerHour: perHour, 186 + PerDay: perDay, 187 + } 188 + s.Limiters[pdsID] = lim 189 + } 190 + 191 + lim.PerSecond.SetLimit(perSecLimit) 192 + lim.PerHour.SetLimit(perHourLimit) 193 + lim.PerDay.SetLimit(perDayLimit) 148 194 } 149 195 150 196 // Shutdown shuts down the slurper ··· 278 324 return !s.newSubsDisabled 279 325 } 280 326 281 - func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool) error { 327 + func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, overrideTrustList bool) error { 282 328 // TODO: for performance, lock on the hostname instead of global 283 329 s.lk.Lock() 284 330 defer s.lk.Unlock() 285 331 286 - if !s.canSlurpHost(host) { 287 - return ErrNewSubsDisabled 288 - } 289 - 290 332 _, ok := s.active[host] 291 333 if ok { 292 334 return nil ··· 302 344 } 303 345 304 346 if peering.ID == 0 { 347 + if !overrideTrustList && !s.canSlurpHost(host) { 348 + return ErrNewSubsDisabled 349 + } 305 350 // New PDS! 306 351 npds := models.PDS{ 307 - Host: host, 308 - SSL: s.ssl, 309 - Registered: reg, 310 - RateLimit: float64(s.DefaultLimit), 311 - CrawlRateLimit: float64(s.DefaultCrawlLimit), 352 + Host: host, 353 + SSL: s.ssl, 354 + Registered: reg, 355 + RateLimit: float64(s.DefaultPerSecondLimit), 356 + HourlyEventLimit: s.DefaultPerHourLimit, 357 + DailyEventLimit: s.DefaultPerDayLimit, 358 + CrawlRateLimit: float64(s.DefaultCrawlLimit), 359 + RepoLimit: s.DefaultRepoLimit, 312 360 } 313 361 if err := s.db.Create(&npds).Error; err != nil { 314 362 return err ··· 332 380 } 333 381 s.active[host] = &sub 334 382 335 - s.GetOrCreateLimiter(peering.ID, peering.RateLimit) 383 + s.GetOrCreateLimiters(peering.ID, int64(peering.RateLimit), peering.HourlyEventLimit, peering.DailyEventLimit) 336 384 337 385 go s.subscribeWithRedialer(ctx, &peering, &sub) 338 386 ··· 360 408 s.active[pds.Host] = &sub 361 409 362 410 // Check if we've already got a limiter for this PDS 363 - s.GetOrCreateLimiter(pds.ID, pds.RateLimit) 411 + s.GetOrCreateLimiters(pds.ID, int64(pds.RateLimit), pds.HourlyEventLimit, pds.DailyEventLimit) 364 412 go s.subscribeWithRedialer(ctx, &pds, &sub) 365 413 } 366 414 ··· 532 580 }, 533 581 } 534 582 535 - limiter := s.GetOrCreateLimiter(host.ID, host.RateLimit) 583 + lims := s.GetOrCreateLimiters(host.ID, int64(host.RateLimit), host.HourlyEventLimit, host.DailyEventLimit) 536 584 537 - instrumentedRSC := events.NewInstrumentedRepoStreamCallbacks(limiter, rsc.EventHandler) 585 + limiters := []*slidingwindow.Limiter{ 586 + lims.PerSecond, 587 + lims.PerHour, 588 + lims.PerDay, 589 + } 590 + 591 + instrumentedRSC := events.NewInstrumentedRepoStreamCallbacks(limiters, rsc.EventHandler) 538 592 539 593 pool := parallel.NewScheduler( 540 594 100,
+1 -1
bgs/handlers.go
··· 162 162 // Maybe we could do something with this response later 163 163 _ = desc 164 164 165 - return s.slurper.SubscribeToPds(ctx, host, true) 165 + return s.slurper.SubscribeToPds(ctx, host, true, false) 166 166 } 167 167 168 168 func (s *BGS) handleComAtprotoSyncNotifyOfUpdate(ctx context.Context, body *comatprototypes.SyncNotifyOfUpdate_Input) error {
+2
cmd/bigsky/docker-compose.yml
··· 10 10 target: /var/lib/postgresql/data 11 11 restart: always 12 12 environment: 13 + POSTGRES_USER: bgs 13 14 POSTGRES_PASSWORD: 33pAstcHDMszLedQah2EVYNgnxbCP 15 + POSTGRES_DB: bgs
+18 -3
cmd/bigsky/main.go
··· 145 145 Value: 4 * time.Hour, 146 146 Usage: "interval between compaction runs, set to 0 to disable scheduled compaction", 147 147 }, 148 + &cli.StringFlag{ 149 + Name: "resolve-address", 150 + EnvVars: []string{"RESOLVE_ADDRESS"}, 151 + Value: "1.1.1.1:53", 152 + }, 153 + &cli.BoolFlag{ 154 + Name: "force-dns-udp", 155 + EnvVars: []string{"FORCE_DNS_UDP"}, 156 + }, 157 + &cli.IntFlag{ 158 + Name: "max-fetch-concurrency", 159 + Value: 100, 160 + EnvVars: []string{"MAX_FETCH_CONCURRENCY"}, 161 + }, 148 162 } 149 163 150 164 app.Action = Bigsky ··· 289 303 290 304 notifman := &notifs.NullNotifs{} 291 305 292 - rf := indexer.NewRepoFetcher(db, repoman) 306 + rf := indexer.NewRepoFetcher(db, repoman, cctx.Int("max-fetch-concurrency")) 293 307 294 308 ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, rf, true, cctx.Bool("spidering"), cctx.Bool("aggregation")) 295 309 if err != nil { ··· 298 312 299 313 rlskip := os.Getenv("BSKY_SOCIAL_RATE_LIMIT_SKIP") 300 314 ix.ApplyPDSClientSettings = func(c *xrpc.Client) { 301 - if c.Host == "https://bsky.social" { 315 + if strings.HasSuffix(c.Host, ".bsky.network") { 302 316 if c.Client == nil { 303 317 c.Client = util.RobustHTTPClient() 304 318 } ··· 310 324 } 311 325 } 312 326 } 327 + rf.ApplyPDSClientSettings = ix.ApplyPDSClientSettings 313 328 314 329 repoman.SetEventHandler(func(ctx context.Context, evt *repomgr.RepoEvent) { 315 330 if err := ix.HandleRepoEvent(ctx, evt); err != nil { ··· 322 337 blobstore = &blobs.DiskBlobStore{Dir: bsdir} 323 338 } 324 339 325 - prodHR, err := api.NewProdHandleResolver(100_000) 340 + prodHR, err := api.NewProdHandleResolver(100_000, cctx.String("resolve-address"), cctx.Bool("force-dns-udp")) 326 341 if err != nil { 327 342 return fmt.Errorf("failed to set up handle resolver: %w", err) 328 343 }
+32 -7
events/consumer.go
··· 7 7 "net" 8 8 "time" 9 9 10 + "github.com/RussellLuo/slidingwindow" 10 11 comatproto "github.com/bluesky-social/indigo/api/atproto" 11 12 "github.com/prometheus/client_golang/prometheus" 12 - "golang.org/x/time/rate" 13 13 14 14 "github.com/gorilla/websocket" 15 15 ) ··· 49 49 } 50 50 51 51 type InstrumentedRepoStreamCallbacks struct { 52 - limiter *rate.Limiter 53 - Next func(ctx context.Context, xev *XRPCStreamEvent) error 52 + limiters []*slidingwindow.Limiter 53 + Next func(ctx context.Context, xev *XRPCStreamEvent) error 54 54 } 55 55 56 - func NewInstrumentedRepoStreamCallbacks(limiter *rate.Limiter, next func(ctx context.Context, xev *XRPCStreamEvent) error) *InstrumentedRepoStreamCallbacks { 56 + func NewInstrumentedRepoStreamCallbacks(limiters []*slidingwindow.Limiter, next func(ctx context.Context, xev *XRPCStreamEvent) error) *InstrumentedRepoStreamCallbacks { 57 57 return &InstrumentedRepoStreamCallbacks{ 58 - limiter: limiter, 59 - Next: next, 58 + limiters: limiters, 59 + Next: next, 60 + } 61 + } 62 + 63 + func waitForLimiter(ctx context.Context, lim *slidingwindow.Limiter) error { 64 + if lim.Allow() { 65 + return nil 66 + } 67 + 68 + // wait until the limiter is ready (check every 100ms) 69 + t := time.NewTicker(100 * time.Millisecond) 70 + defer t.Stop() 71 + 72 + for !lim.Allow() { 73 + select { 74 + case <-ctx.Done(): 75 + return ctx.Err() 76 + case <-t.C: 77 + } 60 78 } 79 + 80 + return nil 61 81 } 62 82 63 83 func (rsc *InstrumentedRepoStreamCallbacks) EventHandler(ctx context.Context, xev *XRPCStreamEvent) error { 64 - rsc.limiter.Wait(ctx) 84 + // Wait on all limiters before calling the next handler 85 + for _, lim := range rsc.limiters { 86 + if err := waitForLimiter(ctx, lim); err != nil { 87 + return err 88 + } 89 + } 65 90 return rsc.Next(ctx, xev) 66 91 } 67 92
+2
go.mod
··· 4 4 5 5 require ( 6 6 contrib.go.opencensus.io/exporter/prometheus v0.4.2 7 + github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b 7 8 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de 8 9 github.com/brianvoe/gofakeit/v6 v6.25.0 9 10 github.com/carlmjohnson/versioninfo v0.22.5 ··· 77 78 78 79 require ( 79 80 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 81 + github.com/go-redis/redis v6.15.9+incompatible // indirect 80 82 github.com/hashicorp/golang-lru v1.0.2 // indirect 81 83 github.com/jackc/puddle/v2 v2.2.1 // indirect 82 84 github.com/klauspost/compress v1.17.3 // indirect
+4 -2
go.sum
··· 35 35 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= 36 36 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 37 37 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 38 + github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU= 39 + github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b/go.mod h1:4+EPqMRApwwE/6yo6CxiHoSnBzjRr3jsqer7frxP8y4= 38 40 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 39 41 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 40 42 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= ··· 143 145 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 144 146 github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= 145 147 github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= 148 + github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= 149 + github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 146 150 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 147 151 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 148 152 github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= ··· 634 638 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= 635 639 github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= 636 640 github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= 637 - github.com/whyrusleeping/cbor-gen v0.0.0-20240104201801-075d1573fac9 h1:973JQTSOMo66VlNZ2+tMQYruE0Yny9DrKvIkr/ybRJg= 638 - github.com/whyrusleeping/cbor-gen v0.0.0-20240104201801-075d1573fac9/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= 639 641 github.com/whyrusleeping/cbor-gen v0.0.0-20240201211319-bf2168ca937c h1:QNbN8SzRc40MGwnd2op/l3E32M445kVJqvgt7NagF4c= 640 642 github.com/whyrusleeping/cbor-gen v0.0.0-20240201211319-bf2168ca937c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= 641 643 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
+1 -1
indexer/indexer.go
··· 68 68 } 69 69 70 70 if crawl { 71 - c, err := NewCrawlDispatcher(fetcher.FetchAndIndexRepo, 10) 71 + c, err := NewCrawlDispatcher(fetcher.FetchAndIndexRepo, fetcher.MaxConcurrency) 72 72 if err != nil { 73 73 return nil, err 74 74 }
+1 -1
indexer/posts_test.go
··· 61 61 62 62 didr := testPLC(t) 63 63 64 - rf := NewRepoFetcher(maindb, repoman) 64 + rf := NewRepoFetcher(maindb, repoman, 10) 65 65 66 66 ix, err := NewIndexer(maindb, notifman, evtman, didr, rf, false, true, true) 67 67 if err != nil {
+4 -1
indexer/repofetch.go
··· 17 17 "gorm.io/gorm" 18 18 ) 19 19 20 - func NewRepoFetcher(db *gorm.DB, rm *repomgr.RepoManager) *RepoFetcher { 20 + func NewRepoFetcher(db *gorm.DB, rm *repomgr.RepoManager, maxConcurrency int) *RepoFetcher { 21 21 return &RepoFetcher{ 22 22 repoman: rm, 23 23 db: db, 24 24 Limiters: make(map[uint]*rate.Limiter), 25 25 ApplyPDSClientSettings: func(*xrpc.Client) {}, 26 + MaxConcurrency: maxConcurrency, 26 27 } 27 28 } 28 29 ··· 32 33 33 34 Limiters map[uint]*rate.Limiter 34 35 LimitMux sync.RWMutex 36 + 37 + MaxConcurrency int 35 38 36 39 ApplyPDSClientSettings func(*xrpc.Client) 37 40 }
+13 -6
models/models.go
··· 104 104 type PDS struct { 105 105 gorm.Model 106 106 107 - Host string 108 - Did string 109 - SSL bool 110 - Cursor int64 111 - Registered bool 112 - Blocked bool 107 + Host string 108 + Did string 109 + SSL bool 110 + Cursor int64 111 + Registered bool 112 + Blocked bool 113 + 113 114 RateLimit float64 114 115 CrawlRateLimit float64 116 + 117 + RepoCount int64 118 + RepoLimit int64 119 + 120 + HourlyEventLimit int64 121 + DailyEventLimit int64 115 122 } 116 123 117 124 func ClientForPds(pds *PDS) *xrpc.Client {
+1 -1
pds/server.go
··· 76 76 repoman := repomgr.NewRepoManager(cs, kmgr) 77 77 notifman := notifs.NewNotificationManager(db, repoman.GetRecord) 78 78 79 - rf := indexer.NewRepoFetcher(db, repoman) 79 + rf := indexer.NewRepoFetcher(db, repoman, 10) 80 80 81 81 ix, err := indexer.NewIndexer(db, notifman, evtman, didr, rf, false, true, true) 82 82 if err != nil {
+8
testing/integ_test.go
··· 38 38 b1.tr.TrialHosts = []string{p1.RawHost()} 39 39 40 40 p1.RequestScraping(t, b1) 41 + p1.BumpLimits(t, b1) 41 42 42 43 time.Sleep(time.Millisecond * 50) 43 44 ··· 134 135 b1.tr.TrialHosts = []string{p1.RawHost(), p2.RawHost()} 135 136 136 137 p1.RequestScraping(t, b1) 138 + p1.BumpLimits(t, b1) 137 139 time.Sleep(time.Millisecond * 100) 138 140 139 141 var users []*TestUser ··· 163 165 time.Sleep(time.Second) 164 166 165 167 p2.RequestScraping(t, b1) 168 + p2.BumpLimits(t, b1) 166 169 time.Sleep(time.Millisecond * 50) 167 170 168 171 // Now, the bgs will discover a gap, and have to catch up somehow ··· 200 203 b1.tr.TrialHosts = []string{p1.RawHost(), p2.RawHost()} 201 204 202 205 p1.RequestScraping(t, b1) 206 + p1.BumpLimits(t, b1) 203 207 time.Sleep(time.Millisecond * 250) 204 208 205 209 users := []*TestUser{p1.MustNewUser(t, usernames[0]+".pdsuno")} ··· 226 230 time.Sleep(time.Second) 227 231 228 232 p2.RequestScraping(t, b1) 233 + p2.BumpLimits(t, b1) 229 234 time.Sleep(time.Second * 2) 230 235 231 236 // Now, the bgs will discover a gap, and have to catch up somehow ··· 256 261 b1.tr.TrialHosts = []string{p1.RawHost()} 257 262 258 263 p1.RequestScraping(t, b1) 264 + p1.BumpLimits(t, b1) 259 265 time.Sleep(time.Millisecond * 50) 260 266 261 267 evts := b1.Events(t, -1) ··· 294 300 b1.tr.TrialHosts = []string{p1.RawHost()} 295 301 296 302 p1.RequestScraping(t, b1) 303 + p1.BumpLimits(t, b1) 297 304 298 305 time.Sleep(time.Millisecond * 50) 299 306 es1 := b1.Events(t, 0) ··· 415 422 b1.tr.TrialHosts = []string{p1.RawHost()} 416 423 417 424 p1.RequestScraping(t, b1) 425 + p1.BumpLimits(t, b1) 418 426 419 427 time.Sleep(time.Millisecond * 50) 420 428
+52 -1
testing/utils.go
··· 1 1 package testing 2 2 3 3 import ( 4 + "bytes" 4 5 "context" 5 6 "crypto/ecdsa" 6 7 "crypto/elliptic" 7 8 "crypto/rand" 8 9 "encoding/base32" 10 + "encoding/json" 9 11 "fmt" 10 12 mathrand "math/rand" 11 13 "net" ··· 168 170 c := &xrpc.Client{Host: "http://" + b.Host()} 169 171 if err := atproto.SyncRequestCrawl(context.TODO(), c, &atproto.SyncRequestCrawl_Input{Hostname: tp.RawHost()}); err != nil { 170 172 t.Fatal(err) 173 + } 174 + } 175 + 176 + func (tp *TestPDS) BumpLimits(t *testing.T, b *TestBGS) { 177 + t.Helper() 178 + 179 + err := b.bgs.CreateAdminToken("test") 180 + if err != nil { 181 + t.Fatal(err) 182 + } 183 + 184 + u, err := url.Parse(tp.HTTPHost()) 185 + if err != nil { 186 + t.Fatal(err) 187 + } 188 + 189 + limReqBody := bgs.RateLimitChangeRequest{ 190 + Host: u.Host, 191 + PerSecond: 5_000, 192 + PerHour: 100_000, 193 + PerDay: 1_000_000, 194 + RepoLimit: 500_000, 195 + CrawlRate: 50_000, 196 + } 197 + 198 + // JSON encode the request body 199 + reqBody, err := json.Marshal(limReqBody) 200 + if err != nil { 201 + t.Fatal(err) 202 + } 203 + 204 + req, err := http.NewRequest("POST", "http://"+b.Host()+"/admin/pds/changeLimits", bytes.NewBuffer(reqBody)) 205 + if err != nil { 206 + t.Fatal(err) 207 + } 208 + req.Header.Set("Content-Type", "application/json") 209 + req.Header.Set("Authorization", "Bearer test") 210 + 211 + // Send the request 212 + client := &http.Client{} 213 + resp, err := client.Do(req) 214 + if err != nil { 215 + t.Fatal(err) 216 + } 217 + defer resp.Body.Close() 218 + 219 + // Check the response 220 + if resp.StatusCode != http.StatusOK { 221 + t.Fatal("expected 200 OK, got: ", resp.Status) 171 222 } 172 223 } 173 224 ··· 437 488 diskpersist, err := events.NewDiskPersistence(filepath.Join(dir, "dp-primary"), filepath.Join(dir, "dp-archive"), maindb, opts) 438 489 439 490 evtman := events.NewEventManager(diskpersist) 440 - rf := indexer.NewRepoFetcher(maindb, repoman) 491 + rf := indexer.NewRepoFetcher(maindb, repoman, 10) 441 492 442 493 ix, err := indexer.NewIndexer(maindb, notifman, evtman, didr, rf, true, true, true) 443 494 if err != nil {
+1 -1
ts/bgs-dash/index.html
··· 4 4 <meta charset="UTF-8" /> 5 5 <!-- <link rel="icon" type="image/svg+xml" href="/vite.svg" /> --> 6 6 <meta name="viewport" content="width=device-width, initial-scale=1.0" /> 7 - <title>BGS Dashboard</title> 7 + <title>Relay Dashboard</title> 8 8 </head> 9 9 <body> 10 10 <div id="root"></div>
+19 -3
ts/bgs-dash/src/App.tsx
··· 14 14 import Domains from "./components/Domains/Domains"; 15 15 import Repos from "./components/Repos/Repos"; 16 16 import Consumers from "./components/Consumers/Consumers"; 17 + import NewPDS from "./components/NewPDS/NewPDS"; 17 18 18 19 function classNames(...classes: string[]) { 19 20 return classes.filter(Boolean).join(" "); ··· 57 58 requrieAuth: true, 58 59 }, 59 60 { 61 + path: "/new_pds", 62 + name: "New PDS", 63 + element: ( 64 + <RequireAuth> 65 + <Nav /> 66 + <main> 67 + <div className="mx-auto max-w-7xl px-2 py-6 sm:px-6 lg:px-8"> 68 + <NewPDS /> 69 + </div> 70 + </main> 71 + </RequireAuth> 72 + ), 73 + }, 74 + { 60 75 path: "/consumers", 61 76 name: "Consumers", 62 77 element: ( ··· 101 116 ), 102 117 requrieAuth: true, 103 118 }, 119 + 104 120 { 105 121 path: "/login", 106 122 name: "Login", ··· 151 167 <img 152 168 className="h-8 w-8" 153 169 src="https://tailwindui.com/img/logos/mark.svg?color=indigo&shade=500" 154 - alt="BGS Admin Dashboard" 170 + alt="Relay Admin Dashboard" 155 171 /> 156 172 </div> 157 173 <div className="hidden md:block"> 158 174 <div className="ml-10 flex items-baseline space-x-4"> 159 175 {routes.map((item) => 160 176 (isAuthed && item.hideIfAuth) || 161 - (!isAuthed && item.requrieAuth) ? null : ( 177 + (!isAuthed && item.requrieAuth) ? null : ( 162 178 <NavLink 163 179 key={item.path} 164 180 to={item.path || "/"} ··· 204 220 <div className="space-y-1 px-2 pb-3 pt-2 sm:px-3"> 205 221 {routes.map((item) => 206 222 (isAuthed && item.hideIfAuth) || 207 - (!isAuthed && item.requrieAuth) ? null : ( 223 + (!isAuthed && item.requrieAuth) ? null : ( 208 224 <Disclosure.Button 209 225 key={item.path} 210 226 className={classNames(
+23 -28
ts/bgs-dash/src/components/Consumers/Consumers.tsx
··· 5 5 NotificationType, 6 6 } from "../Notification/Notification"; 7 7 8 - import { BGS_HOST } from "../../constants"; 8 + import { RELAY_HOST } from "../../constants"; 9 9 10 10 import { useNavigate } from "react-router-dom"; 11 11 import { Consumer, ConsumerKey, ConsumerResponse } from "../../models/consumer"; ··· 51 51 }, []); 52 52 53 53 const refreshPDSList = () => { 54 - fetch(`${BGS_HOST}/admin/consumers/list`, { 54 + fetch(`${RELAY_HOST}/admin/consumers/list`, { 55 55 method: "GET", 56 56 headers: { 57 57 "Content-Type": "application/json", ··· 152 152 Consumer Connections 153 153 </h1> 154 154 <p className="mt-2 text-sm text-gray-700"> 155 - A list of all websocket consumers actively connected to the BGS 155 + A list of all websocket consumers actively connected to the Relay 156 156 </p> 157 157 </div> 158 158 </div> ··· 175 175 > 176 176 ID 177 177 <span 178 - className={`ml-2 flex-none rounded text-gray-400 ${ 179 - sortField === "ID" 180 - ? "group-hover:bg-gray-200" 181 - : "invisible group-hover:visible group-focus:visible" 182 - }`} 178 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "ID" 179 + ? "group-hover:bg-gray-200" 180 + : "invisible group-hover:visible group-focus:visible" 181 + }`} 183 182 > 184 183 {sortField === "ID" && sortOrder === "asc" ? ( 185 184 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> ··· 206 205 > 207 206 Remote Address 208 207 <span 209 - className={`ml-2 flex-none rounded text-gray-400 ${ 210 - sortField === "RemoteAddr" 211 - ? "group-hover:bg-gray-200" 212 - : "invisible group-hover:visible group-focus:visible" 213 - }`} 208 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "RemoteAddr" 209 + ? "group-hover:bg-gray-200" 210 + : "invisible group-hover:visible group-focus:visible" 211 + }`} 214 212 > 215 213 {sortField === "RemoteAddr" && sortOrder === "asc" ? ( 216 214 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> ··· 237 235 > 238 236 User Agent 239 237 <span 240 - className={`ml-2 flex-none rounded text-gray-400 ${ 241 - sortField === "UserAgent" 242 - ? "group-hover:bg-gray-200" 243 - : "invisible group-hover:visible group-focus:visible" 244 - }`} 238 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "UserAgent" 239 + ? "group-hover:bg-gray-200" 240 + : "invisible group-hover:visible group-focus:visible" 241 + }`} 245 242 > 246 243 {sortField === "UserAgent" && sortOrder === "asc" ? ( 247 244 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> ··· 268 265 > 269 266 Events Consumed 270 267 <span 271 - className={`ml-2 flex-none rounded text-gray-400 ${ 272 - sortField === "EventsConsumed" 273 - ? "group-hover:bg-gray-200" 274 - : "invisible group-hover:visible group-focus:visible" 275 - }`} 268 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "EventsConsumed" 269 + ? "group-hover:bg-gray-200" 270 + : "invisible group-hover:visible group-focus:visible" 271 + }`} 276 272 > 277 273 {sortField === "EventsConsumed" && sortOrder === "asc" ? ( 278 274 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> ··· 299 295 > 300 296 Connected At 301 297 <span 302 - className={`ml-2 flex-none rounded text-gray-400 ${ 303 - sortField === "ConnectedAt" 304 - ? "group-hover:bg-gray-200" 305 - : "invisible group-hover:visible group-focus:visible" 306 - }`} 298 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "ConnectedAt" 299 + ? "group-hover:bg-gray-200" 300 + : "invisible group-hover:visible group-focus:visible" 301 + }`} 307 302 > 308 303 {sortField === "ConnectedAt" && sortOrder === "asc" ? ( 309 304 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" />
+320 -116
ts/bgs-dash/src/components/Dash/Dash.tsx
··· 1 1 import { 2 + MagnifyingGlassIcon, 2 3 ShieldCheckIcon, 3 4 ShieldExclamationIcon, 4 5 } from "@heroicons/react/24/outline"; ··· 18 19 NotificationType, 19 20 } from "../Notification/Notification"; 20 21 21 - import { BGS_HOST } from "../../constants"; 22 + import { RELAY_HOST } from "../../constants"; 22 23 import { PDS, PDSKey } from "../../models/pds"; 23 24 24 25 import { Switch } from "@headlessui/react"; ··· 31 32 32 33 const Dash: FC<{}> = () => { 33 34 const [pdsList, setPDSList] = useState<PDS[] | null>(null); 35 + const [fullPDSList, setFullPDSList] = useState<PDS[] | null>(null); 34 36 const [sortField, setSortField] = useState<PDSKey>("ID"); 35 37 const [sortOrder, setSortOrder] = useState<string>("asc"); 36 38 ··· 51 53 type: "block" | "disconnect"; 52 54 pds: PDS; 53 55 } | null>(null); 54 - const [modalConfirm, setModalConfirm] = useState<() => void>(() => {}); 55 - const [modalCancel, setModalCancel] = useState<() => void>(() => {}); 56 + const [modalConfirm, setModalConfirm] = useState<() => void>(() => { }); 57 + const [modalCancel, setModalCancel] = useState<() => void>(() => { }); 56 58 57 - const [editingIngestRateLimit, setEditingIngestRateLimit] = 59 + const [editingPerSecondRateLimit, setEditingPerSecondRateLimimt] = 60 + useState<PDS | null>(null); 61 + const [editingPerHourRateLimit, setEditingPerHourRateLimit] = 62 + useState<PDS | null>(null); 63 + const [editingPerDayRateLimit, setEditingPerDayRateLimit] = 58 64 useState<PDS | null>(null); 59 65 const [editingCrawlRateLimit, setEditingCrawlRateLimit] = 60 66 useState<PDS | null>(null); 67 + const [editingRepoLimit, setEditingRepoLimit] = 68 + useState<PDS | null>(null); 69 + 70 + const [searchTerm, setSearchTerm] = useState<string | undefined>(undefined); 71 + 72 + const filterPDSList = (list: PDS[]): PDS[] => { 73 + // Filter the hostnames based on the search term 74 + if (searchTerm) { 75 + // Support RegEx search 76 + try { 77 + const regex = new RegExp(searchTerm, "i"); 78 + list = list.filter((pds) => { 79 + return regex.test(pds.Host); 80 + }); 81 + } catch (e) { 82 + // If the regex is invalid, just do a normal search 83 + list = list.filter((pds) => { 84 + return pds.Host.toLowerCase().includes(searchTerm.toLowerCase()); 85 + }); 86 + } 87 + } 88 + 89 + return list; 90 + }; 61 91 62 92 const [adminToken, setAdminToken] = useState<string>( 63 93 localStorage.getItem("admin_route_token") || "" ··· 87 117 }, []); 88 118 89 119 useEffect(() => { 90 - document.title = "BGS Admin Dashboard"; 120 + document.title = "Relay Admin Dashboard"; 91 121 }, []); 92 122 93 123 const refreshPDSList = () => { 94 - fetch(`${BGS_HOST}/admin/pds/list`, { 124 + fetch(`${RELAY_HOST}/admin/pds/list`, { 95 125 method: "GET", 96 126 headers: { 97 127 "Content-Type": "application/json", ··· 108 138 ); 109 139 return; 110 140 } 111 - const sortedList = sortPDSList(res); 112 - setPDSList(sortedList); 141 + setFullPDSList(res); 113 142 }) 114 143 .catch((err) => { 115 144 setAlertWithTimeout( ··· 121 150 }; 122 151 123 152 const getSlurpsEnabled = () => { 124 - fetch(`${BGS_HOST}/admin/subs/getEnabled`, { 153 + fetch(`${RELAY_HOST}/admin/subs/getEnabled`, { 125 154 method: "GET", 126 155 headers: { 127 156 "Content-Type": "application/json", ··· 151 180 152 181 const requestSlurpsEnabledStateChange = (state: boolean) => { 153 182 setCanToggleSlurps(false); 154 - fetch(`${BGS_HOST}/admin/subs/setEnabled?enabled=${state}`, { 183 + fetch(`${RELAY_HOST}/admin/subs/setEnabled?enabled=${state}`, { 155 184 method: "POST", 156 185 headers: { 157 186 "Content-Type": "application/json", ··· 186 215 }; 187 216 188 217 const requestCrawlHost = (host: string) => { 189 - fetch(`${BGS_HOST}/xrpc/com.atproto.sync.requestCrawl`, { 218 + fetch(`${RELAY_HOST}/xrpc/com.atproto.sync.requestCrawl`, { 190 219 method: "POST", 191 220 headers: { 192 221 "Content-Type": "application/json", ··· 210 239 211 240 const requestDisconnectHost = (host: string, shouldBlock: boolean) => { 212 241 fetch( 213 - `${BGS_HOST}/admin/subs/killUpstream?host=${host}&block=${shouldBlock}`, 242 + `${RELAY_HOST}/admin/subs/killUpstream?host=${host}&block=${shouldBlock}`, 214 243 { 215 244 method: "POST", 216 245 headers: { ··· 222 251 if (res.status !== 200) { 223 252 setAlertWithTimeout( 224 253 "failure", 225 - `Failed to request ${shouldBlock ? "block" : "disconnect"}: ${ 226 - res.statusText 254 + `Failed to request ${shouldBlock ? "block" : "disconnect"}: ${res.statusText 227 255 } (${res.status})`, 228 256 true 229 257 ); ··· 239 267 }; 240 268 241 269 const requestBlockHost = (host: string) => { 242 - fetch(`${BGS_HOST}/admin/pds/block?host=${host}`, { 270 + fetch(`${RELAY_HOST}/admin/pds/block?host=${host}`, { 243 271 method: "POST", 244 272 headers: { 245 273 "Content-Type": "application/json", ··· 259 287 }); 260 288 }; 261 289 const requestUnblockHost = (host: string) => { 262 - fetch(`${BGS_HOST}/admin/pds/unblock?host=${host}`, { 290 + fetch(`${RELAY_HOST}/admin/pds/unblock?host=${host}`, { 263 291 method: "POST", 264 292 headers: { 265 293 "Content-Type": "application/json", ··· 279 307 }); 280 308 }; 281 309 282 - const changeIngestRateLimit = (pds: PDS, newLimit: number) => { 283 - fetch( 284 - `${BGS_HOST}/admin/pds/changeIngestRateLimit?host=${pds.Host}&limit=${newLimit}`, 285 - { 286 - method: "POST", 287 - headers: { 288 - "Content-Type": "application/json", 289 - Authorization: `Bearer ${adminToken}`, 290 - }, 291 - } 292 - ).then((res) => { 293 - if (res.status !== 200) { 294 - setAlertWithTimeout( 295 - "failure", 296 - `Failed to change rate limit: ${res.statusText} (${res.status})`, 297 - true 298 - ); 299 - } else { 300 - setAlertWithTimeout("success", "Successfully changed rate limit", true); 301 - } 302 - refreshPDSList(); 303 - }); 304 - }; 305 - 306 - const changeCrawlRateLimit = (pds: PDS, newLimit: number) => { 310 + const updateRateLimits = (pds: PDS) => { 307 311 fetch( 308 - `${BGS_HOST}/admin/pds/changeCrawlRateLimit?host=${pds.Host}&limit=${newLimit}`, 312 + `${RELAY_HOST}/admin/pds/changeLimits`, 309 313 { 310 314 method: "POST", 311 315 headers: { 312 316 "Content-Type": "application/json", 313 317 Authorization: `Bearer ${adminToken}`, 314 318 }, 319 + body: JSON.stringify({ 320 + host: pds.Host, 321 + per_second: pds.PerSecondEventRate.Max, 322 + per_hour: pds.PerHourEventRate.Max, 323 + per_day: pds.PerDayEventRate.Max, 324 + crawl_rate: pds.CrawlRate.Max, 325 + repo_limit: pds.RepoLimit, 326 + }), 315 327 } 316 328 ).then((res) => { 317 329 if (res.status !== 200) { ··· 321 333 true 322 334 ); 323 335 } else { 324 - setAlertWithTimeout("success", "Successfully changed rate limit", true); 336 + setAlertWithTimeout("success", "Successfully changed rate limits", true); 325 337 } 326 338 refreshPDSList(); 327 339 }); ··· 377 389 }; 378 390 379 391 useEffect(() => { 380 - if (!pdsList) { 392 + if (!fullPDSList) { 381 393 return; 382 394 } 383 - setPDSList(sortPDSList(pdsList)); 384 - }, [sortOrder, sortField]); 395 + setPDSList(sortPDSList(filterPDSList(fullPDSList!))); 396 + }, [sortOrder, sortField, searchTerm, fullPDSList]); 385 397 386 398 useEffect(() => { 387 399 refreshPDSList(); ··· 421 433 A list of all PDS connections and their current status. 422 434 </p> 423 435 </div> 436 + 424 437 <div className="inline-flex mt-5 sm:mt-0"> 425 438 <Switch.Group as="div" className="flex items-center justify-between"> 426 439 <span className="flex flex-grow flex-col mr-5"> ··· 461 474 </Switch> 462 475 </Switch.Group> 463 476 </div> 477 + 478 + </div> 479 + <div className="flex flex-1 items-center justify-center py-2 lg:justify-start"> 480 + <div className="w-full max-w-lg lg:max-w-xs"> 481 + <label htmlFor="search" className="sr-only"> 482 + Search 483 + </label> 484 + <div className="relative"> 485 + <div className="pointer-events-none absolute inset-y-0 left-0 flex items-center pl-3"> 486 + <MagnifyingGlassIcon className="h-5 w-5 text-gray-400" aria-hidden="true" /> 487 + </div> 488 + <input 489 + id="search" 490 + name="search" 491 + className="block w-full rounded-md border-0 bg-white py-1.5 pl-10 pr-3 text-gray-900 ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6" 492 + placeholder="Search" 493 + type="search" 494 + onChange={(e) => { 495 + setSearchTerm(e.target.value); 496 + }} 497 + value={searchTerm || ""} 498 + /> 499 + </div> 500 + </div> 464 501 </div> 465 502 466 503 <div className="mt-8 flow-root"> ··· 482 519 > 483 520 ID 484 521 <span 485 - className={`ml-2 flex-none rounded text-gray-400 ${ 486 - sortField === "ID" 487 - ? "group-hover:bg-gray-200" 488 - : "invisible group-hover:visible group-focus:visible" 489 - }`} 522 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "ID" 523 + ? "group-hover:bg-gray-200" 524 + : "invisible group-hover:visible group-focus:visible" 525 + }`} 490 526 > 491 527 {sortField === "ID" && sortOrder === "asc" ? ( 492 528 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> ··· 513 549 > 514 550 Host 515 551 <span 516 - className={`ml-2 flex-none rounded text-gray-400 ${ 517 - sortField === "Host" 518 - ? "group-hover:bg-gray-200" 519 - : "invisible group-hover:visible group-focus:visible" 520 - }`} 552 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "Host" 553 + ? "group-hover:bg-gray-200" 554 + : "invisible group-hover:visible group-focus:visible" 555 + }`} 521 556 > 522 557 {sortField === "Host" && sortOrder === "asc" ? ( 523 558 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> ··· 544 579 > 545 580 Connected 546 581 <span 547 - className={`ml-2 flex-none rounded text-gray-400 ${ 548 - sortField === "HasActiveConnection" 549 - ? "group-hover:bg-gray-200" 550 - : "invisible group-hover:visible group-focus:visible" 551 - }`} 582 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "HasActiveConnection" 583 + ? "group-hover:bg-gray-200" 584 + : "invisible group-hover:visible group-focus:visible" 585 + }`} 552 586 > 553 587 {sortField === "HasActiveConnection" && 554 - sortOrder === "asc" ? ( 588 + sortOrder === "asc" ? ( 555 589 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> 556 590 ) : ( 557 591 <ChevronDownIcon ··· 576 610 > 577 611 Permitted 578 612 <span 579 - className={`ml-2 flex-none rounded text-gray-400 ${ 580 - sortField === "Blocked" 581 - ? "group-hover:bg-gray-200" 582 - : "invisible group-hover:visible group-focus:visible" 583 - }`} 613 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "Blocked" 614 + ? "group-hover:bg-gray-200" 615 + : "invisible group-hover:visible group-focus:visible" 616 + }`} 584 617 > 585 618 {sortField === "Blocked" && sortOrder === "asc" ? ( 586 619 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> ··· 601 634 href="#" 602 635 className="group inline-flex" 603 636 onClick={() => { 604 - setSortField("UserCount"); 637 + setSortField("RepoCount"); 605 638 setSortOrder(sortOrder === "asc" ? "desc" : "asc"); 606 639 }} 607 640 > 608 641 Users 609 642 <span 610 - className={`ml-2 flex-none rounded text-gray-400 ${ 611 - sortField === "UserCount" 612 - ? "group-hover:bg-gray-200" 613 - : "invisible group-hover:visible group-focus:visible" 614 - }`} 643 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "RepoCount" 644 + ? "group-hover:bg-gray-200" 645 + : "invisible group-hover:visible group-focus:visible" 646 + }`} 615 647 > 616 - {sortField === "UserCount" && sortOrder === "asc" ? ( 648 + {sortField === "RepoCount" && sortOrder === "asc" ? ( 617 649 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> 618 650 ) : ( 619 651 <ChevronDownIcon ··· 638 670 > 639 671 Events Seen 640 672 <span 641 - className={`ml-2 flex-none rounded text-gray-400 ${ 642 - sortField === "EventsSeenSinceStartup" 643 - ? "group-hover:bg-gray-200" 644 - : "invisible group-hover:visible group-focus:visible" 645 - }`} 673 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "EventsSeenSinceStartup" 674 + ? "group-hover:bg-gray-200" 675 + : "invisible group-hover:visible group-focus:visible" 676 + }`} 646 677 > 647 678 {sortField === "EventsSeenSinceStartup" && 648 - sortOrder === "asc" ? ( 679 + sortOrder === "asc" ? ( 649 680 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> 650 681 ) : ( 651 682 <ChevronDownIcon ··· 670 701 > 671 702 Cursor 672 703 <span 673 - className={`ml-2 flex-none rounded text-gray-400 ${ 674 - sortField === "Cursor" 675 - ? "group-hover:bg-gray-200" 676 - : "invisible group-hover:visible group-focus:visible" 677 - }`} 704 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "Cursor" 705 + ? "group-hover:bg-gray-200" 706 + : "invisible group-hover:visible group-focus:visible" 707 + }`} 678 708 > 679 709 {sortField === "Cursor" && sortOrder === "asc" ? ( 680 710 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> ··· 692 722 className="px-3 py-3.5 text-right text-sm font-semibold text-gray-900 pr-6 whitespace-nowrap" 693 723 > 694 724 <a href="#" className="group inline-flex"> 695 - Ingest Rate Limit 725 + Events Per Second Limit 726 + </a> 727 + </th> 728 + <th 729 + scope="col" 730 + className="px-3 py-3.5 text-right text-sm font-semibold text-gray-900 pr-6 whitespace-nowrap" 731 + > 732 + <a href="#" className="group inline-flex"> 733 + Per Hour Limit 696 734 </a> 697 735 </th> 698 736 <th ··· 700 738 className="px-3 py-3.5 text-right text-sm font-semibold text-gray-900 pr-6 whitespace-nowrap" 701 739 > 702 740 <a href="#" className="group inline-flex"> 703 - Tokens 741 + Per Day Limit 704 742 </a> 705 743 </th> 706 744 <th ··· 716 754 className="px-3 py-3.5 text-right text-sm font-semibold text-gray-900 pr-6 whitespace-nowrap" 717 755 > 718 756 <a href="#" className="group inline-flex"> 719 - Tokens 757 + Repo Limit 720 758 </a> 721 759 </th> 722 760 <th ··· 733 771 > 734 772 First Seen 735 773 <span 736 - className={`ml-2 flex-none rounded text-gray-400 ${ 737 - sortField === "CreatedAt" 738 - ? "group-hover:bg-gray-200" 739 - : "invisible group-hover:visible group-focus:visible" 740 - }`} 774 + className={`ml-2 flex-none rounded text-gray-400 ${sortField === "CreatedAt" 775 + ? "group-hover:bg-gray-200" 776 + : "invisible group-hover:visible group-focus:visible" 777 + }`} 741 778 > 742 779 {sortField === "CreatedAt" && sortOrder === "asc" ? ( 743 780 <ChevronUpIcon className="h-5 w-5" aria-hidden="true" /> ··· 842 879 )} 843 880 </td> 844 881 <td className="whitespace-nowrap px-3 py-2 text-sm text-gray-400 text-center w-8 pr-6"> 845 - {pds.UserCount?.toLocaleString()} 882 + {pds.RepoCount?.toLocaleString()} 846 883 </td> 847 884 <td className="whitespace-nowrap px-3 py-2 text-sm text-gray-400 text-center w-8 pr-6"> 848 885 {pds.EventsSeenSinceStartup?.toLocaleString()} ··· 853 890 <td className="whitespace-nowrap px-3 py-2 text-sm text-gray-400 text-center w-8 pr-6"> 854 891 <span 855 892 className={ 856 - editingIngestRateLimit?.ID === pds.ID 893 + editingPerSecondRateLimit?.ID === pds.ID 857 894 ? "hidden" 858 895 : "" 859 896 } 860 897 > 861 - {pds.IngestRate.MaxEventsPerSecond?.toLocaleString()} 898 + {pds.PerSecondEventRate.Max?.toLocaleString()} 862 899 /sec 863 900 </span> 864 901 <input 865 902 type="number" 866 - name={`ingest-rate-limit-${pds.ID}`} 867 - id={`ingest-rate-limit-${pds.ID}`} 903 + name={`per-second-rate-limit-${pds.ID}`} 904 + id={`per-second-rate-limit-${pds.ID}`} 868 905 className={ 869 906 `inline-block w-24 rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6` + 870 - (editingIngestRateLimit?.ID === pds.ID 907 + (editingPerSecondRateLimit?.ID === pds.ID 871 908 ? "" 872 909 : " hidden") 873 910 } 874 - defaultValue={pds.IngestRate.MaxEventsPerSecond?.toLocaleString()} 911 + defaultValue={pds.PerSecondEventRate.Max?.toLocaleString()} 875 912 /> 876 913 <a 877 914 href="#" 878 - onClick={() => setEditingIngestRateLimit(pds)} 879 - className={editingIngestRateLimit ? "hidden" : ""} 915 + onClick={() => setEditingPerSecondRateLimimt(pds)} 916 + className={editingPerSecondRateLimit ? "hidden" : ""} 880 917 > 881 918 <PencilSquareIcon 882 919 className="h-5 w-5 text-gray-500 ml-1 inline-block align-sub" ··· 887 924 href="#" 888 925 onClick={() => { 889 926 const newRateLimit = document.getElementById( 890 - `ingest-rate-limit-${pds.ID}` 927 + `per-second-rate-limit-${pds.ID}` 891 928 ) as HTMLInputElement; 892 929 if (newRateLimit) { 893 - changeIngestRateLimit(pds, +newRateLimit.value); 930 + pds.PerSecondEventRate.Max = +newRateLimit.value; 931 + updateRateLimits(pds); 894 932 } 895 - setEditingIngestRateLimit(null); 933 + setEditingPerSecondRateLimimt(null); 896 934 }} 897 935 className={ 898 936 "rounded-md p-2 ml-1 hover:text-green-600 hover:bg-green-100 focus:outline-none focus:ring-2 focus:ring-green-600 focus:ring-offset-2 focus:ring-offset-green-50" + 899 - (editingIngestRateLimit?.ID === pds.ID 937 + (editingPerSecondRateLimit?.ID === pds.ID 900 938 ? "" 901 939 : " hidden") 902 940 } ··· 908 946 </a> 909 947 </td> 910 948 <td className="whitespace-nowrap px-3 py-2 text-sm text-gray-400 text-center w-8 pr-6"> 911 - {Math.abs( 912 - pds.IngestRate.TokenCount || 0 913 - ).toLocaleString()} 949 + <span 950 + className={ 951 + editingPerHourRateLimit?.ID === pds.ID 952 + ? "hidden" 953 + : "" 954 + } 955 + > 956 + {pds.PerHourEventRate.Max?.toLocaleString()} 957 + /hour 958 + </span> 959 + <input 960 + type="number" 961 + name={`per-hour-rate-limit-${pds.ID}`} 962 + id={`per-hour-rate-limit-${pds.ID}`} 963 + className={ 964 + `inline-block w-24 rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6` + 965 + (editingPerHourRateLimit?.ID === pds.ID 966 + ? "" 967 + : " hidden") 968 + } 969 + defaultValue={pds.PerHourEventRate.Max?.toLocaleString()} 970 + /> 971 + <a 972 + href="#" 973 + onClick={() => setEditingPerHourRateLimit(pds)} 974 + className={editingPerHourRateLimit ? "hidden" : ""} 975 + > 976 + <PencilSquareIcon 977 + className="h-5 w-5 text-gray-500 ml-1 inline-block align-sub" 978 + aria-hidden="true" 979 + /> 980 + </a> 981 + <a 982 + href="#" 983 + onClick={() => { 984 + const newRateLimit = document.getElementById( 985 + `per-hour-rate-limit-${pds.ID}` 986 + ) as HTMLInputElement; 987 + if (newRateLimit) { 988 + pds.PerHourEventRate.Max = +newRateLimit.value; 989 + updateRateLimits(pds); 990 + } 991 + setEditingPerHourRateLimit(null); 992 + }} 993 + className={ 994 + "rounded-md p-2 ml-1 hover:text-green-600 hover:bg-green-100 focus:outline-none focus:ring-2 focus:ring-green-600 focus:ring-offset-2 focus:ring-offset-green-50" + 995 + (editingPerHourRateLimit?.ID === pds.ID 996 + ? "" 997 + : " hidden") 998 + } 999 + > 1000 + <CheckIcon 1001 + className="h-5 w-5 text-green-500 inline-block align-sub" 1002 + aria-hidden="true" 1003 + /> 1004 + </a> 914 1005 </td> 915 1006 <td className="whitespace-nowrap px-3 py-2 text-sm text-gray-400 text-center w-8 pr-6"> 916 1007 <span 917 1008 className={ 918 - editingCrawlRateLimit?.ID === pds.ID ? "hidden" : "" 1009 + editingPerDayRateLimit?.ID === pds.ID 1010 + ? "hidden" 1011 + : "" 919 1012 } 920 1013 > 921 - {pds.CrawlRate.MaxEventsPerSecond?.toLocaleString()} 1014 + {pds.PerDayEventRate.Max?.toLocaleString()} 1015 + /day 1016 + </span> 1017 + <input 1018 + type="number" 1019 + name={`per-day-limit-${pds.ID}`} 1020 + id={`per-day-rate-limit-${pds.ID}`} 1021 + className={ 1022 + `inline-block w-24 rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6` + 1023 + (editingPerDayRateLimit?.ID === pds.ID 1024 + ? "" 1025 + : " hidden") 1026 + } 1027 + defaultValue={pds.PerDayEventRate.Max?.toLocaleString()} 1028 + /> 1029 + <a 1030 + href="#" 1031 + onClick={() => setEditingPerDayRateLimit(pds)} 1032 + className={editingPerDayRateLimit ? "hidden" : ""} 1033 + > 1034 + <PencilSquareIcon 1035 + className="h-5 w-5 text-gray-500 ml-1 inline-block align-sub" 1036 + aria-hidden="true" 1037 + /> 1038 + </a> 1039 + <a 1040 + href="#" 1041 + onClick={() => { 1042 + const newRateLimit = document.getElementById( 1043 + `per-day-rate-limit-${pds.ID}` 1044 + ) as HTMLInputElement; 1045 + if (newRateLimit) { 1046 + pds.PerDayEventRate.Max = +newRateLimit.value; 1047 + updateRateLimits(pds); 1048 + } 1049 + setEditingPerDayRateLimit(null); 1050 + }} 1051 + className={ 1052 + "rounded-md p-2 ml-1 hover:text-green-600 hover:bg-green-100 focus:outline-none focus:ring-2 focus:ring-green-600 focus:ring-offset-2 focus:ring-offset-green-50" + 1053 + (editingPerDayRateLimit?.ID === pds.ID 1054 + ? "" 1055 + : " hidden") 1056 + } 1057 + > 1058 + <CheckIcon 1059 + className="h-5 w-5 text-green-500 inline-block align-sub" 1060 + aria-hidden="true" 1061 + /> 1062 + </a> 1063 + </td> 1064 + <td className="whitespace-nowrap px-3 py-2 text-sm text-gray-400 text-center w-8 pr-6"> 1065 + <span 1066 + className={ 1067 + editingCrawlRateLimit?.ID === pds.ID 1068 + ? "hidden" 1069 + : "" 1070 + } 1071 + > 1072 + {pds.CrawlRate.Max?.toLocaleString()} 922 1073 /sec 923 1074 </span> 924 1075 <input ··· 931 1082 ? "" 932 1083 : " hidden") 933 1084 } 934 - defaultValue={pds.CrawlRate.MaxEventsPerSecond?.toLocaleString()} 1085 + defaultValue={pds.CrawlRate.Max?.toLocaleString()} 935 1086 /> 936 1087 <a 937 1088 href="#" ··· 950 1101 `crawl-rate-limit-${pds.ID}` 951 1102 ) as HTMLInputElement; 952 1103 if (newRateLimit) { 953 - changeCrawlRateLimit(pds, +newRateLimit.value); 1104 + pds.CrawlRate.Max = +newRateLimit.value; 1105 + updateRateLimits(pds); 954 1106 } 955 1107 setEditingCrawlRateLimit(null); 956 1108 }} ··· 968 1120 </a> 969 1121 </td> 970 1122 <td className="whitespace-nowrap px-3 py-2 text-sm text-gray-400 text-center w-8 pr-6"> 971 - {Math.abs( 972 - pds.CrawlRate.TokenCount || 0 973 - ).toLocaleString()} 1123 + <span 1124 + className={ 1125 + editingRepoLimit?.ID === pds.ID 1126 + ? "hidden" 1127 + : "" 1128 + } 1129 + > 1130 + {pds.RepoLimit?.toLocaleString()} 1131 + </span> 1132 + <input 1133 + type="number" 1134 + name={`repo-limit-${pds.ID}`} 1135 + id={`repo-limit-${pds.ID}`} 1136 + className={ 1137 + `inline-block w-24 rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6` + 1138 + (editingRepoLimit?.ID === pds.ID 1139 + ? "" 1140 + : " hidden") 1141 + } 1142 + defaultValue={pds.RepoLimit?.toLocaleString()} 1143 + /> 1144 + <a 1145 + href="#" 1146 + onClick={() => setEditingRepoLimit(pds)} 1147 + className={editingRepoLimit ? "hidden" : ""} 1148 + > 1149 + <PencilSquareIcon 1150 + className="h-5 w-5 text-gray-500 ml-1 inline-block align-sub" 1151 + aria-hidden="true" 1152 + /> 1153 + </a> 1154 + <a 1155 + href="#" 1156 + onClick={() => { 1157 + const newLimit = document.getElementById( 1158 + `repo-limit-${pds.ID}` 1159 + ) as HTMLInputElement; 1160 + if (newLimit) { 1161 + pds.RepoLimit = +newLimit.value; 1162 + updateRateLimits(pds); 1163 + } 1164 + setEditingRepoLimit(null); 1165 + }} 1166 + className={ 1167 + "rounded-md p-2 ml-1 hover:text-green-600 hover:bg-green-100 focus:outline-none focus:ring-2 focus:ring-green-600 focus:ring-offset-2 focus:ring-offset-green-50" + 1168 + (editingRepoLimit?.ID === pds.ID 1169 + ? "" 1170 + : " hidden") 1171 + } 1172 + > 1173 + <CheckIcon 1174 + className="h-5 w-5 text-green-500 inline-block align-sub" 1175 + aria-hidden="true" 1176 + /> 1177 + </a> 974 1178 </td> 975 1179 <td className="whitespace-nowrap px-3 py-2 text-sm text-gray-400 text-center w-8 pr-6"> 976 1180 {new Date(Date.parse(pds.CreatedAt)).toLocaleString()}
+7 -7
ts/bgs-dash/src/components/Domains/Domains.tsx
··· 9 9 NotificationType, 10 10 } from "../Notification/Notification"; 11 11 12 - import { BGS_HOST } from "../../constants"; 12 + import { RELAY_HOST } from "../../constants"; 13 13 14 14 import { useNavigate } from "react-router-dom"; 15 15 import ConfirmDomainBanModal from "./ConfirmDomainBanModal"; ··· 37 37 domain: string; 38 38 type: "ban" | "unban"; 39 39 } | null>(null); 40 - const [modalConfirm, setModalConfirm] = useState<() => void>(() => {}); 41 - const [modalCancel, setModalCancel] = useState<() => void>(() => {}); 40 + const [modalConfirm, setModalConfirm] = useState<() => void>(() => { }); 41 + const [modalCancel, setModalCancel] = useState<() => void>(() => { }); 42 42 43 43 const [adminToken, setAdminToken] = useState<string>( 44 44 localStorage.getItem("admin_route_token") || "" ··· 68 68 }, []); 69 69 70 70 useEffect(() => { 71 - document.title = "BGS Admin Dashboard"; 71 + document.title = "Relay Admin Dashboard"; 72 72 }, []); 73 73 74 74 const refreshDomainBanList = () => { 75 - fetch(`${BGS_HOST}/admin/subs/listDomainBans`, { 75 + fetch(`${RELAY_HOST}/admin/subs/listDomainBans`, { 76 76 method: "GET", 77 77 headers: { 78 78 "Content-Type": "application/json", ··· 102 102 }; 103 103 104 104 const requestBanDomain = (domain: string) => { 105 - fetch(`${BGS_HOST}/admin/subs/banDomain`, { 105 + fetch(`${RELAY_HOST}/admin/subs/banDomain`, { 106 106 method: "POST", 107 107 headers: { 108 108 "Content-Type": "application/json", ··· 135 135 }; 136 136 137 137 const requestUnbanDomain = (domain: string) => { 138 - fetch(`${BGS_HOST}/admin/subs/unbanDomain`, { 138 + fetch(`${RELAY_HOST}/admin/subs/unbanDomain`, { 139 139 method: "POST", 140 140 headers: { 141 141 "Content-Type": "application/json",
+3 -3
ts/bgs-dash/src/components/Login/Login.tsx
··· 1 1 import React, { useState } from "react"; 2 2 import { useNavigate } from "react-router-dom"; 3 - import { BGS_HOST } from "../../constants"; 3 + import { RELAY_HOST } from "../../constants"; 4 4 import Notification, { NotificationMeta } from "../Notification/Notification"; 5 5 6 6 export default function Login() { ··· 20 20 21 21 if (token) { 22 22 // Try to make a request to the Admin API to verify the token 23 - fetch(`${BGS_HOST}/admin/pds/list`, { 23 + fetch(`${RELAY_HOST}/admin/pds/list`, { 24 24 method: "GET", 25 25 headers: { 26 26 "Content-Type": "application/json", ··· 74 74 )} 75 75 </div> 76 76 <h2 className=" text-center text-2xl font-bold leading-9 tracking-tight text-gray-900"> 77 - Login to the BGS Admin Dashboard 77 + Login to the Relay Admin Dashboard 78 78 </h2> 79 79 </div> 80 80
+107
ts/bgs-dash/src/components/NewPDS/ConfirmNewPDSModal.tsx
··· 1 + import { Fragment, useState } from "react"; 2 + import { Dialog, Transition } from "@headlessui/react"; 3 + import { XCircleIcon } from "@heroicons/react/24/outline"; 4 + 5 + interface ConfirmModalProps { 6 + action: { 7 + type: "add" | "remove"; 8 + pds: string; 9 + }; 10 + onConfirm: () => void; 11 + onCancel: () => void; 12 + } 13 + 14 + const ConfirmNewPDSModal = ({ 15 + action, 16 + onConfirm, 17 + onCancel, 18 + }: ConfirmModalProps) => { 19 + const [open, setOpen] = useState(true); 20 + 21 + const handleConfirm = () => { 22 + onConfirm(); 23 + setOpen(false); 24 + }; 25 + 26 + const handleCancel = () => { 27 + onCancel(); 28 + setOpen(false); 29 + }; 30 + 31 + return ( 32 + <Transition.Root show={open} as={Fragment}> 33 + <Dialog as="div" className="relative z-10" onClose={setOpen}> 34 + <Transition.Child 35 + as={Fragment} 36 + enter="ease-out duration-300" 37 + enterFrom="opacity-0" 38 + enterTo="opacity-100" 39 + leave="ease-in duration-200" 40 + leaveFrom="opacity-100" 41 + leaveTo="opacity-0" 42 + > 43 + <div className="fixed inset-0 bg-gray-500 bg-opacity-75 transition-opacity" /> 44 + </Transition.Child> 45 + 46 + <div className="fixed inset-0 z-10 overflow-y-auto"> 47 + <div className="flex min-h-full items-center justify-center p-4 text-center sm:items-center sm:p-0"> 48 + <Transition.Child 49 + as={Fragment} 50 + enter="ease-out duration-300" 51 + enterFrom="opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95" 52 + enterTo="opacity-100 translate-y-0 sm:scale-100" 53 + leave="ease-in duration-200" 54 + leaveFrom="opacity-100 translate-y-0 sm:scale-100" 55 + leaveTo="opacity-0 translate-y-4 sm:translate-y-0 sm:scale-95" 56 + > 57 + <Dialog.Panel className="relative transform overflow-hidden rounded-lg bg-white px-4 pt-5 pb-4 text-left shadow-xl transition-all sm:my-8 sm:w-full sm:max-w-sm sm:p-6"> 58 + <div> 59 + <div className="mx-auto flex h-12 w-12 items-center justify-center rounded-full bg-yellow-100"> 60 + <XCircleIcon 61 + className="h-6 w-6 text-yellow-600" 62 + aria-hidden="true" 63 + /> 64 + </div> 65 + <div className="mt-3 text-center sm:mt-5"> 66 + <Dialog.Title 67 + as="h3" 68 + className="text-lg font-medium leading-6 text-gray-900" 69 + > 70 + {`${action.type[0].toLocaleUpperCase()}${action.type.substring( 71 + 1 72 + )}`}{" "} 73 + PDS 74 + </Dialog.Title> 75 + <div className="mt-2"> 76 + <p className="text-sm text-gray-500"> 77 + Are you sure you want to {action.type} {action.pds}? 78 + </p> 79 + </div> 80 + </div> 81 + </div> 82 + <div className="mt-5 sm:mt-6 sm:grid sm:grid-cols-2 sm:gap-3 sm:grid-flow-row-dense"> 83 + <button 84 + type="button" 85 + className="inline-flex w-full justify-center rounded-md bg-indigo-600 px-3 py-2 text-sm font-medium text-white shadow-sm hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-indigo-500 focus:ring-offset-2 sm:col-start-2" 86 + onClick={handleConfirm} 87 + > 88 + Confirm 89 + </button> 90 + <button 91 + type="button" 92 + className="mt-3 inline-flex w-full justify-center rounded-md bg-white px-3 py-2 text-sm font-medium text-gray-700 shadow-sm ring-1 ring-inset ring-gray-300 hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-indigo-500 sm:col-start-1 sm:mt-0" 93 + onClick={handleCancel} 94 + > 95 + Cancel 96 + </button> 97 + </div> 98 + </Dialog.Panel> 99 + </Transition.Child> 100 + </div> 101 + </div> 102 + </Dialog> 103 + </Transition.Root> 104 + ); 105 + }; 106 + 107 + export default ConfirmNewPDSModal;
+243
ts/bgs-dash/src/components/NewPDS/NewPDS.tsx
··· 1 + import { FC, useEffect, useState } from "react"; 2 + import Notification, { 3 + NotificationMeta, 4 + NotificationType, 5 + } from "../Notification/Notification"; 6 + 7 + import { RELAY_HOST } from "../../constants"; 8 + 9 + import { useNavigate } from "react-router-dom"; 10 + import ConfirmNewPDSModal from "./ConfirmNewPDSModal"; 11 + import { 12 + ShieldCheckIcon, 13 + } from "@heroicons/react/24/outline"; 14 + 15 + const NewPDS: FC<{}> = () => { 16 + const [pdsHost, setPDSHost] = useState<string>(""); 17 + 18 + // Notification Management 19 + const [shouldShowNotification, setShouldShowNotification] = 20 + useState<boolean>(false); 21 + const [notification, setNotification] = useState<NotificationMeta>({ 22 + message: "", 23 + alertType: "", 24 + }); 25 + 26 + // Modal state management 27 + const [modalAction, setModalAction] = useState<{ 28 + pds: string; 29 + type: "add" | "remove"; 30 + } | null>(null); 31 + const [modalConfirm, setModalConfirm] = useState<() => void>(() => { }); 32 + const [modalCancel, setModalCancel] = useState<() => void>(() => { }); 33 + 34 + const [adminToken, setAdminToken] = useState<string>( 35 + localStorage.getItem("admin_route_token") || "" 36 + ); 37 + const navigate = useNavigate(); 38 + 39 + const setAlertWithTimeout = ( 40 + type: NotificationType, 41 + message: string, 42 + dismiss: boolean 43 + ) => { 44 + setNotification({ 45 + message, 46 + alertType: type, 47 + autodismiss: dismiss, 48 + }); 49 + setShouldShowNotification(true); 50 + }; 51 + 52 + useEffect(() => { 53 + const token = localStorage.getItem("admin_route_token"); 54 + if (token) { 55 + setAdminToken(token); 56 + } else { 57 + navigate("/login"); 58 + } 59 + }, []); 60 + 61 + const requestAddPDS = (pds: string) => { 62 + fetch(`${RELAY_HOST}/admin/pds/requestCrawl`, { 63 + method: "POST", 64 + headers: { 65 + "Content-Type": "application/json", 66 + Authorization: `Bearer ${adminToken}`, 67 + }, 68 + body: JSON.stringify({ 69 + hostname: pds, 70 + }), 71 + }) 72 + .then((res) => { 73 + if (res.status !== 200) { 74 + try { 75 + res.json().then((data) => { 76 + if (data.error) { 77 + setAlertWithTimeout( 78 + "failure", 79 + `Failed to add PDS: ${data.error}`, 80 + true 81 + ); 82 + } else { 83 + setAlertWithTimeout( 84 + "failure", 85 + `Failed to add PDS: ${res.statusText}`, 86 + true 87 + ); 88 + } 89 + }); 90 + } 91 + catch (err) { 92 + setAlertWithTimeout( 93 + "failure", 94 + `Failed to add PDS: ${err}`, 95 + true 96 + ); 97 + } 98 + } else { 99 + try { 100 + res.json().then((data) => { 101 + if (data.error) { 102 + setAlertWithTimeout( 103 + "failure", 104 + `Failed to add PDS: ${data.error}`, 105 + true 106 + ); 107 + } else { 108 + setAlertWithTimeout( 109 + "success", 110 + `Successfully added PDS ${pds}`, 111 + true 112 + ); 113 + } 114 + }); 115 + } catch (err) { 116 + setAlertWithTimeout( 117 + "failure", 118 + `Failed to add PDS: ${err}`, 119 + true 120 + ); 121 + } 122 + } 123 + }) 124 + .catch((err) => { 125 + setAlertWithTimeout("failure", `Failed to add PDS: ${err}`, true); 126 + }); 127 + }; 128 + 129 + const requestRemovePDS = (pds: string) => { 130 + setAlertWithTimeout( 131 + "failure", 132 + `Failed to remove PDS: ${pds} - Not implemented`, 133 + true 134 + ); 135 + }; 136 + 137 + const handleAddPDS = ( 138 + pds: string, 139 + type: "add" | "remove" 140 + ) => { 141 + if (pds === "") { 142 + setAlertWithTimeout("failure", "PDS Hostname cannot be empty", true); 143 + return; 144 + } 145 + 146 + // Strip the protocol from the hostname 147 + pds = pds.replace(/^https?:\/\//, ""); 148 + 149 + setModalAction({ pds: pds, type }); 150 + 151 + setModalConfirm(() => { 152 + return () => { 153 + if (type === "add") requestAddPDS(pds); 154 + else requestRemovePDS(pds); 155 + 156 + setModalAction(null); 157 + }; 158 + }); 159 + 160 + setModalCancel(() => { 161 + return () => { 162 + setModalAction(null); 163 + }; 164 + }); 165 + }; 166 + 167 + return ( 168 + <div className="mx-auto max-w-full"> 169 + {shouldShowNotification ? ( 170 + <Notification 171 + message={notification.message} 172 + alertType={notification.alertType} 173 + subMessage={notification.subMessage} 174 + autodismiss={notification.autodismiss} 175 + unshow={() => { 176 + setShouldShowNotification(false); 177 + setNotification({ message: "", alertType: "" }); 178 + }} 179 + show={shouldShowNotification} 180 + ></Notification> 181 + ) : ( 182 + <></> 183 + )} 184 + <div className="sm:flex sm:items-center"> 185 + <div className="sm:flex-auto"> 186 + <h1 className="text-2xl font-semibold leading-6 text-gray-900"> 187 + Add a PDS 188 + </h1> 189 + <p className="mt-2 text-sm text-gray-700"> 190 + Add a PDS to the Relay and trigger crawling. 191 + </p> 192 + </div> 193 + </div> 194 + <div className="flex-grow mt-5"> 195 + <div className="max-w-3xl w-full"> 196 + <label 197 + htmlFor="email" 198 + className="block text-sm font-medium leading-6 text-gray-900" 199 + > 200 + PDS Hostname 201 + </label> 202 + <div className="mt-2 inline-flex flex-col sm:flex-row"> 203 + <input 204 + type="text" 205 + name="pds" 206 + id="pds" 207 + className="block w-72 rounded-md border-0 py-1.5 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6" 208 + placeholder="hydnum.us-west.host.bsky.network" 209 + value={pdsHost} 210 + onChange={(e) => { 211 + setPDSHost(e.target.value); 212 + }} 213 + /> 214 + <div className="inline-flex mt-4 sm:mt-0"> 215 + <button 216 + type="button" 217 + onClick={() => { 218 + handleAddPDS(pdsHost.trim(), "add"); 219 + }} 220 + className="ml-0 sm:ml-2 inline-flex whitespace-nowrap items-center gap-x-1.5 rounded-md bg-green-600 px-2.5 py-1.5 text-sm font-semibold text-white shadow-sm hover:bg-green-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-green-600" 221 + > 222 + <ShieldCheckIcon 223 + className="-ml-0.5 h-5 w-5" 224 + aria-hidden="true" 225 + /> 226 + Add PDS 227 + </button> 228 + </div> 229 + </div> 230 + </div> 231 + </div> 232 + {modalAction && ( 233 + <ConfirmNewPDSModal 234 + action={modalAction} 235 + onConfirm={modalConfirm} 236 + onCancel={modalCancel} 237 + /> 238 + )} 239 + </div> 240 + ); 241 + }; 242 + 243 + export default NewPDS;
+6 -6
ts/bgs-dash/src/components/Repos/Repos.tsx
··· 4 4 NotificationType, 5 5 } from "../Notification/Notification"; 6 6 7 - import { BGS_HOST } from "../../constants"; 7 + import { RELAY_HOST } from "../../constants"; 8 8 9 9 import { useNavigate } from "react-router-dom"; 10 10 import ConfirmRepoTakedownModal from "./ConfirmRepoTakedownModal"; ··· 29 29 repo: string; 30 30 type: "takedown" | "untakedown"; 31 31 } | null>(null); 32 - const [modalConfirm, setModalConfirm] = useState<() => void>(() => {}); 33 - const [modalCancel, setModalCancel] = useState<() => void>(() => {}); 32 + const [modalConfirm, setModalConfirm] = useState<() => void>(() => { }); 33 + const [modalCancel, setModalCancel] = useState<() => void>(() => { }); 34 34 35 35 const [adminToken, setAdminToken] = useState<string>( 36 36 localStorage.getItem("admin_route_token") || "" ··· 60 60 }, []); 61 61 62 62 const requestTakedownRepo = (repo: string) => { 63 - fetch(`${BGS_HOST}/admin/repo/takeDown`, { 63 + fetch(`${RELAY_HOST}/admin/repo/takeDown`, { 64 64 method: "POST", 65 65 headers: { 66 66 "Content-Type": "application/json", ··· 92 92 }; 93 93 94 94 const requestUntakedownRepo = (repo: string) => { 95 - fetch(`${BGS_HOST}/admin/repo/reverseTakedown`, { 95 + fetch(`${RELAY_HOST}/admin/repo/reverseTakedown`, { 96 96 method: "POST", 97 97 headers: { 98 98 "Content-Type": "application/json", ··· 173 173 Repo Takedowns 174 174 </h1> 175 175 <p className="mt-2 text-sm text-gray-700"> 176 - Takedown a repo to purge it from the BGS history and reject all 176 + Takedown a repo to purge it from the Relay history and reject all 177 177 future events for it. 178 178 </p> 179 179 </div>
+7 -3
ts/bgs-dash/src/constants.ts
··· 1 - const BGS_HOST = `${window.location.protocol}//${window.location.hostname}:${ 2 - window.location.hostname === "localhost" ? "2470" : window.location.port 1 + const isDev = 2 + window.location.hostname === "localhost" || 3 + window.location.hostname === "jaz1"; 4 + 5 + const RELAY_HOST = `${window.location.protocol}//${window.location.hostname}:${ 6 + isDev ? "2470" : window.location.port 3 7 }`; 4 8 5 - export { BGS_HOST }; 9 + export { RELAY_HOST };
+7 -4
ts/bgs-dash/src/models/pds.ts
··· 1 1 interface RateLimit { 2 - MaxEventsPerSecond: number; 3 - TokenCount: number; 2 + Max: number; 3 + WindowSeconds: number; 4 4 } 5 5 6 6 interface PDS { ··· 16 16 Blocked: boolean; 17 17 HasActiveConnection: boolean; 18 18 EventsSeenSinceStartup?: number; 19 - IngestRate: RateLimit; 20 19 CrawlRate: RateLimit; 21 - UserCount: number; 20 + PerSecondEventRate: RateLimit; 21 + PerHourEventRate: RateLimit; 22 + PerDayEventRate: RateLimit; 23 + RepoCount: number; 24 + RepoLimit: number; 22 25 } 23 26 24 27 type PDSKey = keyof PDS;
+3
util/http.go
··· 6 6 "net/http" 7 7 "time" 8 8 9 + "github.com/hashicorp/go-cleanhttp" 9 10 "github.com/hashicorp/go-retryablehttp" 11 + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" 10 12 ) 11 13 12 14 type LeveledSlog struct { ··· 45 47 46 48 logger := LeveledSlog{inner: slog.Default().With("subsystem", "RobustHTTPClient")} 47 49 retryClient := retryablehttp.NewClient() 50 + retryClient.HTTPClient.Transport = otelhttp.NewTransport(cleanhttp.DefaultPooledTransport()) 48 51 retryClient.RetryMax = 3 49 52 retryClient.RetryWaitMin = 1 * time.Second 50 53 retryClient.RetryWaitMax = 10 * time.Second