lightweight
com.atproto.sync.listReposByCollection
warn on task exit
+20
-9
+5
-5
hacking.md
+5
-5
hacking.md
···
95
95
- [x] rachet by PDS host: be lenient if we have never seen a sync1.1-looking commit, always strict after we see one.
96
96
- [?] boooo we might need more handling for pre-sync1.1 repos if they don't include adjacent keys
97
97
- [x] split the keyspace: put the rbc/cbr indexes on a second keyspace with larger block size, expect hits on main keyspace
98
-
- [ ] resync free hints from first phony getRecord
99
-
- [ ] short-circuit: tiny repos may incidentally return their entire CAR for getRecord
100
-
- [ ] estimate CAR size and `getRecord` if it's likely very small (bypass `describeRepo`)
98
+
- [ ] firehose websocket
99
+
- [-] ~~ping/pong (unless jacquard is already doing it):~~ seems like no but we can skip it
100
+
- [ ] no-events-received timeout reconnect
101
+
- [ ] resync short-circuit: tiny repos may actually return their entire CAR for getRecord
101
102
- [ ] commit CAR handling: generate a list of keys with gaps noted, to reliably detect missing adjacent keys
102
103
- [ ] account status convergeance: if we receive commits from apparently-inactive accounts, should we check upstream status to make sure we're not stale?
103
-
- [ ] websocket ping/pong (unless jacquard is already doing it)
104
-
- [ ] websocket no-events-received timeout reconnect
105
104
106
105
107
106
very much still todo but i'm getting tired
108
107
- [ ] add a `--heavy` mode that always uses `getRepo` and never `describeRepo`
108
+
- [ ] resync: estimate CAR size from `getRecord` mst height; `getRepo` if it's likely very small
109
109
- [ ] multi-relay subscriber
110
110
- [ ] special did:web behaviour to keep reusing a stale resolution on failure
111
111
- [ ] admin view of backfill state etc
+15
-4
src/main.rs
+15
-4
src/main.rs
···
5
5
use clap::Parser;
6
6
use tokio::task::JoinSet;
7
7
use tokio_util::sync::CancellationToken;
8
-
use tracing::info;
8
+
use tracing::{info, warn};
9
9
10
10
use lightrail::error::{Error, Result};
11
11
use lightrail::identity;
···
147
147
args.max_firehose_workers,
148
148
Duration::from_secs(args.cursor_save_interval_secs),
149
149
);
150
-
sub.run(token).await
150
+
sub.run(token)
151
+
.await
152
+
.inspect_err(|e| warn!(error = %e, "firehose exited"))
151
153
}
152
154
});
153
155
···
157
159
let client = client.clone();
158
160
let host = subscribe_host.clone();
159
161
async move {
160
-
if backfill::run(host, db, client, token.clone(), None).await? {
162
+
if backfill::run(host, db, client, token.clone(), None)
163
+
.await
164
+
.inspect_err(|e| warn!(error = %e, "backfill exited"))?
165
+
{
161
166
info!("backfill task idling");
162
167
token.cancelled().await;
163
168
}
···
181
186
token,
182
187
)
183
188
.await
189
+
.inspect_err(|e| warn!(error = %e, "resync exited"))
184
190
}
185
191
});
186
192
···
194
200
subscribe_host: subscribe_host.clone(),
195
201
admin_password: pw,
196
202
});
197
-
async move { lightrail::server::serve(addr, db, token, admin_config).await }
203
+
async move {
204
+
lightrail::server::serve(addr, db, token, admin_config)
205
+
.await
206
+
.inspect_err(|e| warn!(error = %e, "server exited"))
207
+
}
198
208
});
199
209
200
210
if args.deep_crawl {
···
214
224
resolver,
215
225
)
216
226
.await
227
+
.inspect_err(|e| warn!(error = %e, "deep crawl exited"))
217
228
}
218
229
});
219
230
}