Per-day new PDS limiter to allow less constrained federation (#657)

+27

bgs/admin.go

··· 37 37 }) 38 38 } 39 39 40 + func (bgs *BGS) handleAdminGetNewPDSPerDayRateLimit(e echo.Context) error { 41 + limit := bgs.slurper.GetNewPDSPerDayLimit() 42 + return e.JSON(200, map[string]int64{ 43 + "limit": limit, 44 + }) 45 + } 46 + 47 + func (bgs *BGS) handleAdminSetNewPDSPerDayRateLimit(e echo.Context) error { 48 + limit, err := strconv.ParseInt(e.QueryParam("limit"), 10, 64) 49 + if err != nil { 50 + return &echo.HTTPError{ 51 + Code: 400, 52 + Message: fmt.Errorf("failed to parse limit: %w", err).Error(), 53 + } 54 + } 55 + 56 + err = bgs.slurper.SetNewPDSPerDayLimit(limit) 57 + if err != nil { 58 + return &echo.HTTPError{ 59 + Code: 500, 60 + Message: fmt.Errorf("failed to set new PDS per day rate limit: %w", err).Error(), 61 + } 62 + } 63 + 64 + return nil 65 + } 66 + 40 67 func (bgs *BGS) handleAdminTakeDownRepo(e echo.Context) error { 41 68 ctx := e.Request().Context() 42 69

+2

bgs/bgs.go

··· 331 331 // Slurper-related Admin API 332 332 admin.GET("/subs/getUpstreamConns", bgs.handleAdminGetUpstreamConns) 333 333 admin.GET("/subs/getEnabled", bgs.handleAdminGetSubsEnabled) 334 + admin.GET("/subs/perDayLimit", bgs.handleAdminGetNewPDSPerDayRateLimit) 334 335 admin.POST("/subs/setEnabled", bgs.handleAdminSetSubsEnabled) 335 336 admin.POST("/subs/killUpstream", bgs.handleAdminKillUpstreamConn) 337 + admin.POST("/subs/setPerDayLimit", bgs.handleAdminSetNewPDSPerDayRateLimit) 336 338 337 339 // Domain-related Admin API 338 340 admin.GET("/subs/listDomainBans", bgs.handleAdminListDomainBans)

+32 -4

bgs/fedmgr.go

··· 42 42 DefaultCrawlLimit rate.Limit 43 43 DefaultRepoLimit int64 44 44 45 + NewPDSPerDayLimiter *slidingwindow.Limiter 46 + 45 47 newSubsDisabled bool 46 48 trustedDomains []string 47 49 ··· 222 224 s.newSubsDisabled = sc.NewSubsDisabled 223 225 s.trustedDomains = sc.TrustedDomains 224 226 227 + s.NewPDSPerDayLimiter, _ = slidingwindow.NewLimiter(time.Hour*24, sc.NewPDSPerDayLimit, windowFunc) 228 + 225 229 return nil 226 230 } 227 231 228 232 type SlurpConfig struct { 229 233 gorm.Model 230 234 231 - NewSubsDisabled bool 232 - TrustedDomains pq.StringArray `gorm:"type:text[]"` 235 + NewSubsDisabled bool 236 + TrustedDomains pq.StringArray `gorm:"type:text[]"` 237 + NewPDSPerDayLimit int64 233 238 } 234 239 235 240 func (s *Slurper) SetNewSubsDisabled(dis bool) error { ··· 250 255 return s.newSubsDisabled 251 256 } 252 257 258 + func (s *Slurper) SetNewPDSPerDayLimit(limit int64) error { 259 + s.lk.Lock() 260 + defer s.lk.Unlock() 261 + 262 + if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("new_pds_per_day_limit", limit).Error; err != nil { 263 + return err 264 + } 265 + 266 + s.NewPDSPerDayLimiter.SetLimit(limit) 267 + return nil 268 + } 269 + 270 + func (s *Slurper) GetNewPDSPerDayLimit() int64 { 271 + s.lk.Lock() 272 + defer s.lk.Unlock() 273 + return s.NewPDSPerDayLimiter.Limit() 274 + } 275 + 253 276 func (s *Slurper) AddTrustedDomain(domain string) error { 254 277 s.lk.Lock() 255 278 defer s.lk.Unlock() ··· 306 329 // Checks whether a host is allowed to be subscribed to 307 330 // must be called with the slurper lock held 308 331 func (s *Slurper) canSlurpHost(host string) bool { 332 + // Check if we're over the limit for new PDSs today 333 + if !s.NewPDSPerDayLimiter.Allow() { 334 + return false 335 + } 336 + 309 337 // Check if the host is a trusted domain 310 338 for _, d := range s.trustedDomains { 311 339 // If the domain starts with a *., it's a wildcard ··· 324 352 return !s.newSubsDisabled 325 353 } 326 354 327 - func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, overrideTrustList bool) error { 355 + func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, adminOverride bool) error { 328 356 // TODO: for performance, lock on the hostname instead of global 329 357 s.lk.Lock() 330 358 defer s.lk.Unlock() ··· 344 372 } 345 373 346 374 if peering.ID == 0 { 347 - if !overrideTrustList && !s.canSlurpHost(host) { 375 + if !adminOverride && !s.canSlurpHost(host) { 348 376 return ErrNewSubsDisabled 349 377 } 350 378 // New PDS!

+6

cmd/palomar/docker-compose.yml

··· 14 14 - "plugins.security.disabled=true" 15 15 - "bootstrap.memory_lock=true" # Disable JVM heap memory swapping 16 16 - "OPENSEARCH_JAVA_OPTS=-Xms4096m -Xmx4096m" # Set min and max JVM heap sizes to at least 50% of system RAM 17 + - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=0penSearch-Pal0mar" 17 18 ulimits: 18 19 memlock: 19 20 soft: -1 ··· 43 44 - "PALOMAR_BGS_SYNC_RATE_LIMIT=20" 44 45 - "PALOMAR_INDEX_MAX_CONCURRENCY=5" 45 46 - "DATABASE_URL=sqlite:///data/palomar/search.db" 47 + - "PALOMAR_BIND=:3997" 48 + - "PALOMAR_METRICS_LISTEN=:3996" 46 49 depends_on: 47 50 - opensearch 51 + ports: 52 + - "3997:3997" 53 + - "3996:3996" 48 54 volumes: 49 55 - type: bind 50 56 source: ../../data

+23

testing/utils.go

··· 167 167 func (tp *TestPDS) RequestScraping(t *testing.T, b *TestRelay) { 168 168 t.Helper() 169 169 170 + err := b.bgs.CreateAdminToken("test") 171 + if err != nil { 172 + t.Fatal(err) 173 + } 174 + 175 + req, err := http.NewRequest("POST", "http://"+b.Host()+"/admin/subs/setPerDayLimit?limit=500", nil) 176 + 177 + req.Header.Set("Content-Type", "application/json") 178 + req.Header.Set("Authorization", "Bearer test") 179 + 180 + // Send the request 181 + client := &http.Client{} 182 + resp, err := client.Do(req) 183 + if err != nil { 184 + t.Fatal(err) 185 + } 186 + defer resp.Body.Close() 187 + 188 + // Check the response 189 + if resp.StatusCode != http.StatusOK { 190 + t.Fatal("expected 200 OK, got: ", resp.Status) 191 + } 192 + 170 193 c := &xrpc.Client{Host: "http://" + b.Host()} 171 194 if err := atproto.SyncRequestCrawl(context.TODO(), c, &atproto.SyncRequestCrawl_Input{Hostname: tp.RawHost()}); err != nil { 172 195 t.Fatal(err)

+153 -46

ts/bgs-dash/src/components/Dash/Dash.tsx

··· 39 39 // Slurp Toggle Management 40 40 const [slurpsEnabled, setSlurpsEnabled] = useState<boolean>(true); 41 41 const [canToggleSlurps, setCanToggleSlurps] = useState<boolean>(true); 42 + const [newPDSLimit, setNewPDSLimit] = useState<number>(0); 43 + const [canSetNewPDSLimit, setCanSetNewPDSLimit] = useState<boolean>(true); 42 44 43 45 // Notification Management 44 46 const [shouldShowNotification, setShouldShowNotification] = ··· 214 216 }); 215 217 }; 216 218 219 + const getNewPDSRateLimit = () => { 220 + fetch(`${RELAY_HOST}/admin/subs/perDayLimit`, { 221 + method: "GET", 222 + headers: { 223 + "Content-Type": "application/json", 224 + Authorization: `Bearer ${adminToken}`, 225 + }, 226 + }) 227 + .then((res) => res.json()) 228 + .then((res) => { 229 + if ("error" in res) { 230 + setAlertWithTimeout( 231 + "failure", 232 + `Failed to fetch New PDS rate limit: ${res.error}`, 233 + true 234 + ); 235 + return; 236 + } 237 + setNewPDSLimit(res.limit); 238 + }) 239 + .catch((err) => { 240 + setAlertWithTimeout( 241 + "failure", 242 + `Failed to fetch New PDS rate limit: ${err}`, 243 + true 244 + ); 245 + }); 246 + } 247 + 248 + const setNewPDSRateLimit = (limit: number) => { 249 + setCanSetNewPDSLimit(false); 250 + fetch(`${RELAY_HOST}/admin/subs/setPerDayLimit?limit=${limit}`, { 251 + method: "POST", 252 + headers: { 253 + "Content-Type": "application/json", 254 + Authorization: `Bearer ${adminToken}`, 255 + }, 256 + 257 + }) 258 + .then((res) => { 259 + setCanSetNewPDSLimit(true); 260 + if (res.status !== 200) { 261 + setAlertWithTimeout( 262 + "failure", 263 + `Failed to set New PDS rate limit: ${res.status}`, 264 + true 265 + ); 266 + return; 267 + } 268 + setAlertWithTimeout( 269 + "success", 270 + `Successfully set New PDS rate limit to ${limit} / day`, 271 + true 272 + ); 273 + setNewPDSLimit(limit); 274 + }) 275 + .catch((err) => { 276 + setCanSetNewPDSLimit(true); 277 + setAlertWithTimeout( 278 + "failure", 279 + `Failed to set New PDS rate limit: ${err}`, 280 + true 281 + ); 282 + }); 283 + } 284 + 217 285 const requestCrawlHost = (host: string) => { 218 286 fetch(`${RELAY_HOST}/xrpc/com.atproto.sync.requestCrawl`, { 219 287 method: "POST", ··· 398 466 useEffect(() => { 399 467 refreshPDSList(); 400 468 getSlurpsEnabled(); 469 + getNewPDSRateLimit(); 401 470 // Refresh stats every 10 seconds 402 471 const interval = setInterval(() => { 403 472 refreshPDSList(); 404 473 getSlurpsEnabled(); 474 + getNewPDSRateLimit(); 405 475 }, 10 * 1000); 406 476 407 477 return () => clearInterval(interval); ··· 424 494 ) : ( 425 495 <></> 426 496 )} 497 + <div></div> 427 498 <div className="sm:flex sm:items-center"> 428 499 <div className="sm:flex-auto"> 429 500 <h1 className="text-2xl font-semibold leading-6 text-gray-900"> ··· 433 504 A list of all PDS connections and their current status. 434 505 </p> 435 506 </div> 436 - 437 - <div className="inline-flex mt-5 sm:mt-0"> 438 - <Switch.Group as="div" className="flex items-center justify-between"> 439 - <span className="flex flex-grow flex-col mr-5"> 440 - <Switch.Label as="span" className="text-gray-900" passive> 441 - {slurpsEnabled ? ( 442 - <ShieldCheckIcon 443 - className="h-5 w-5 mr-2 mb-1 inline-block" 444 - aria-hidden="true" 445 - /> 446 - ) : ( 447 - <ShieldExclamationIcon 448 - className="h-5 w-5 mr-2 mb-1 inline-block" 449 - aria-hidden="true" 450 - /> 451 - )} 452 - <span className="text-md font-medium leading-6"> 453 - New Connections {slurpsEnabled ? "Enabled" : "Disabled"} 454 - </span> 455 - </Switch.Label> 456 - </span> 457 - <Switch 458 - checked={slurpsEnabled} 459 - onChange={requestSlurpsEnabledStateChange} 460 - disabled={!canToggleSlurps} 461 - className={classNames( 462 - slurpsEnabled ? "bg-green-600" : "bg-red-400", 463 - canToggleSlurps ? "cursor-pointer" : "cursor-not-allowed", 464 - "relative inline-flex h-6 w-11 flex-shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors duration-200 ease-in-out focus:outline-none focus:ring-2 focus:ring-green-600 focus:ring-offset-2" 465 - )} 466 - > 467 - <span 468 - aria-hidden="true" 507 + <div className="flex flex-col mt-5"> 508 + <div className="inline-flex mt-5 sm:mt-0 flex-col"> 509 + <Switch.Group as="div" className="flex items-center justify-between"> 510 + <span className="flex flex-grow flex-col mr-5"> 511 + <Switch.Label as="span" className="text-gray-900" passive> 512 + {slurpsEnabled ? ( 513 + <ShieldCheckIcon 514 + className="h-5 w-5 mr-2 mb-1 inline-block" 515 + aria-hidden="true" 516 + /> 517 + ) : ( 518 + <ShieldExclamationIcon 519 + className="h-5 w-5 mr-2 mb-1 inline-block" 520 + aria-hidden="true" 521 + /> 522 + )} 523 + <span className="text-md font-medium leading-6"> 524 + New Connections {slurpsEnabled ? "Enabled" : "Disabled"} 525 + </span> 526 + </Switch.Label> 527 + </span> 528 + <Switch 529 + checked={slurpsEnabled} 530 + onChange={requestSlurpsEnabledStateChange} 531 + disabled={!canToggleSlurps} 469 532 className={classNames( 470 - slurpsEnabled ? "translate-x-5" : "translate-x-0", 471 - "pointer-events-none inline-block h-5 w-5 transform rounded-full bg-white shadow ring-0 transition duration-200 ease-in-out" 533 + slurpsEnabled ? "bg-green-600" : "bg-red-400", 534 + canToggleSlurps ? "cursor-pointer" : "cursor-not-allowed", 535 + "relative inline-flex h-6 w-11 flex-shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors duration-200 ease-in-out focus:outline-none focus:ring-2 focus:ring-green-600 focus:ring-offset-2" 472 536 )} 473 - /> 474 - </Switch> 475 - </Switch.Group> 537 + > 538 + <span 539 + aria-hidden="true" 540 + className={classNames( 541 + slurpsEnabled ? "translate-x-5" : "translate-x-0", 542 + "pointer-events-none inline-block h-5 w-5 transform rounded-full bg-white shadow ring-0 transition duration-200 ease-in-out" 543 + )} 544 + /> 545 + </Switch> 546 + </Switch.Group> 547 + </div> 548 + <div className="ml-4"> 549 + <div className="mt-2 flex rounded-md shadow-sm"> 550 + <div className="relative flex flex-grow items-stretch focus-within:z-10"> 551 + <input 552 + type="number" 553 + id="new-pds-rate-limit" 554 + name="new-pds-rate-limit" 555 + // Hides the up/down arrows on number inputs 556 + className="[appearance:textfield] [&::-webkit-outer-spin-button]:appearance-none [&::-webkit-inner-spin-button]:appearance-none block w-full rounded-none rounded-l-md border-0 py-1.5 text-gray-900 ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6" 557 + value={newPDSLimit} 558 + aria-describedby="rate-limit" 559 + onChange={(e) => { 560 + setNewPDSLimit(parseInt(e.target.value)); 561 + }} 562 + /> 563 + <div className="pointer-events-none absolute inset-y-0 right-0 flex items-center pr-3"> 564 + <span className="text-gray-500 sm:text-sm" id="price-currency"> 565 + PDS / Day 566 + </span> 567 + </div> 568 + </div> 569 + <button 570 + type="button" 571 + className="relative -ml-px inline-flex items-center gap-x-1.5 rounded-r-md px-3 py-2 text-sm font-semibold text-gray-900 ring-1 ring-inset ring-gray-300 hover:bg-gray-50" 572 + disabled={!canSetNewPDSLimit} 573 + onClick={() => { 574 + setNewPDSRateLimit(newPDSLimit); 575 + }} 576 + > 577 + Update 578 + </button> 579 + </div> 580 + </div> 476 581 </div> 477 582 478 583 </div> ··· 1186 1291 </table> 1187 1292 </div> 1188 1293 </div> 1189 - {modalAction && ( 1190 - <ConfirmModal 1191 - action={modalAction} 1192 - onConfirm={modalConfirm} 1193 - onCancel={modalCancel} 1194 - /> 1195 - )} 1196 - </div> 1294 + { 1295 + modalAction && ( 1296 + <ConfirmModal 1297 + action={modalAction} 1298 + onConfirm={modalConfirm} 1299 + onCancel={modalCancel} 1300 + /> 1301 + ) 1302 + } 1303 + </div > 1197 1304 ); 1198 1305 }; 1199 1306

Configure Feed

Configure Feed