very fast at protocol indexer with flexible filtering, xrpc queries, cursor-backed event stream, and more, built on fjall
rust
fjall
at-protocol
atproto
indexer
[firehose,crawler] also check io errors if its our fault or not
+23
-6
+4
-4
src/crawler/list_repos.rs
+4
-4
src/crawler/list_repos.rs
···
3
3
use crate::state::AppState;
4
4
use crate::util::throttle::{OrFailure, ThrottleHandle, Throttler};
5
5
use crate::util::{
6
-
ErrorForStatus, RetryOutcome, RetryWithBackoff, WatchEnabledExt, is_status_their_fault,
7
-
is_tls_cert_error, parse_retry_after,
6
+
ErrorForStatus, RetryOutcome, RetryWithBackoff, WatchEnabledExt, is_io_error_their_fault,
7
+
is_status_their_fault, is_tls_cert_error, parse_retry_after,
8
8
};
9
9
use chrono::{DateTime, TimeDelta, Utc};
10
10
use fjall::OwnedWriteBatch;
···
113
113
114
114
let mut src = e.source();
115
115
while let Some(s) = src {
116
-
if let Some(io_err) = s.downcast_ref::<std::io::Error>() {
117
-
if is_tls_cert_error(io_err) {
116
+
if let Some(e) = s.downcast_ref::<std::io::Error>() {
117
+
if is_io_error_their_fault(&e) || is_tls_cert_error(e) {
118
118
return true;
119
119
}
120
120
}
+3
-2
src/ingest/firehose.rs
+3
-2
src/ingest/firehose.rs
···
4
4
use crate::state::AppState;
5
5
use crate::util::throttle::ThrottleHandle;
6
6
use crate::util::{
7
-
WatchEnabledExt, is_status_their_fault, is_timeout, is_tls_cert_error, is_tls_error_their_fault,
7
+
WatchEnabledExt, is_io_error_their_fault, is_status_their_fault, is_timeout, is_tls_cert_error,
8
+
is_tls_error_their_fault,
8
9
};
9
10
use jacquard_common::IntoStatic;
10
11
use jacquard_common::types::did::Did;
···
28
29
29
30
match e {
30
31
WsError::Rustls(e) if is_tls_error_their_fault(e) => return true,
31
-
WsError::Io(io_err) if is_tls_cert_error(io_err) => return true,
32
+
WsError::Io(e) if is_io_error_their_fault(e) || is_tls_cert_error(e) => return true,
32
33
WsError::CannotResolveHost => return true,
33
34
WsError::Upgrade(WsUpgradeError::DidNotSwitchProtocols(status))
34
35
if is_status_their_fault(*status) =>
+16
src/util/mod.rs
+16
src/util/mod.rs
···
49
49
false
50
50
}
51
51
52
+
pub fn is_io_error_their_fault(e: &std::io::Error) -> bool {
53
+
use std::io::ErrorKind::*;
54
+
matches!(
55
+
e.kind(),
56
+
// some of these maybe our fault, but lets assume we have working networking
57
+
// if its our fault chances are most of the other hosts will also fail, which will be in the logs
58
+
// we log the error anyway so it should be easy to tell if something is going bad
59
+
ConnectionRefused
60
+
| HostUnreachable
61
+
| NetworkUnreachable
62
+
| ConnectionReset
63
+
| ConnectionAborted
64
+
| TimedOut
65
+
)
66
+
}
67
+
52
68
pub fn is_tls_error_their_fault(e: &rustls::Error) -> bool {
53
69
use rustls::Error::*;
54
70
matches!(