very fast at protocol indexer with flexible filtering, xrpc queries, cursor-backed event stream, and more, built on fjall
rust
fjall
at-protocol
atproto
indexer
[firehose,crawler] treat 1xx,2xx,3xx,5xx status codes as server error (aside from crawler, where 500 and 2xx is allowed)
+17
-13
+5
-2
src/crawler/list_repos.rs
+5
-2
src/crawler/list_repos.rs
···
121
121
src = s.source();
122
122
}
123
123
124
-
e.status()
125
-
.map_or(false, |s| is_status_their_fault(s.as_u16()))
124
+
e.status().map_or(false, |s| {
125
+
// lets not consider internal server errors for throttling
126
+
// a server might be having a hard time on one request, but not on the rest
127
+
s.as_u16() != 500 && is_status_their_fault(s.as_u16())
128
+
})
126
129
}
127
130
128
131
/// shared describeRepo signal-checking logic used by both relay and retry producers.
+1
-1
src/ingest/firehose.rs
+1
-1
src/ingest/firehose.rs
···
32
32
WsError::Io(e) if is_io_error_their_fault(e) || is_tls_cert_error(e) => return true,
33
33
WsError::CannotResolveHost => return true,
34
34
WsError::Upgrade(WsUpgradeError::DidNotSwitchProtocols(status))
35
-
if is_status_their_fault(*status) =>
35
+
if (*status >= 200 && *status < 300) || is_status_their_fault(*status) =>
36
36
{
37
37
return true;
38
38
}
+11
-10
src/util/mod.rs
+11
-10
src/util/mod.rs
···
87
87
)
88
88
}
89
89
90
+
// use this for public (unauth) xrpc errors
90
91
pub fn is_status_their_fault(status: u16) -> bool {
91
-
return matches!(
92
-
status,
93
-
502 // BAD_GATEWAY
94
-
| 503 // SERVICE_UNAVAILABLE
95
-
| 504 // GATEWAY_TIMEOUT
96
-
| 522 // CONNECTION_TIMEOUT
97
-
| 525 // SSL_HANDSHAKE_FAILURE
98
-
| 530 // SITE_FROZEN
99
-
| 404 // NOT FOUND: we know its not our fault because we use known xrpcs..
100
-
);
92
+
return (status >= 100 && status < 200) // informational, why are we here?
93
+
|| (status >= 500 && status < 600) // server error :>
94
+
|| (status >= 300 && status < 400) // any 3xx error doesnt make sense in the context of a pds / relay
95
+
|| matches!(
96
+
status,
97
+
404 // NOT FOUND: we know its not our fault because we use known xrpcs..
98
+
| 436 // some stupid ass error code idk, some domain park uses this i think???
99
+
| 403 // FORBIDDEN: sob
100
+
| 401 // UNAUTHORIZED: sob
101
+
);
101
102
}
102
103
103
104
/// outcome of [`RetryWithBackoff::retry`] when the operation does not succeed.