Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
dropshot mult-value query param: pain
+132
-11
+14
Cargo.lock
+14
Cargo.lock
···
3149
3149
]
3150
3150
3151
3151
[[package]]
3152
+
name = "serde_qs"
3153
+
version = "1.0.0-rc.3"
3154
+
source = "registry+https://github.com/rust-lang/crates.io-index"
3155
+
checksum = "4cb0b9062a400c31442e67d1f2b1e7746bebd691110ebee1b7d0c7293b04fab1"
3156
+
dependencies = [
3157
+
"itoa",
3158
+
"percent-encoding",
3159
+
"ryu",
3160
+
"serde",
3161
+
"thiserror 2.0.12",
3162
+
]
3163
+
3164
+
[[package]]
3152
3165
name = "serde_spanned"
3153
3166
version = "0.6.8"
3154
3167
source = "registry+https://github.com/rust-lang/crates.io-index"
···
3827
3840
"semver",
3828
3841
"serde",
3829
3842
"serde_json",
3843
+
"serde_qs",
3830
3844
"sha2",
3831
3845
"tempfile",
3832
3846
"thiserror 2.0.12",
+1
ufos/Cargo.toml
+1
ufos/Cargo.toml
+1
ufos/src/lib.rs
+1
ufos/src/lib.rs
+73
ufos/src/qs_query.rs
+73
ufos/src/qs_query.rs
···
1
+
use async_trait::async_trait;
2
+
use dropshot::{
3
+
ApiEndpointBodyContentType, ExclusiveExtractor, ExtractorMetadata, HttpError, RequestContext,
4
+
RequestInfo, ServerContext, SharedExtractor,
5
+
};
6
+
/// copied from https://github.com/oxidecomputer/dropshot/blob/695e1d8872c988c43066eb0848c87c127eeda361/dropshot/src/extractor/query.rs
7
+
/// Apache 2.0: https://github.com/oxidecomputer/dropshot/blob/695e1d8872c988c43066eb0848c87c127eeda361/LICENSE
8
+
use schemars::JsonSchema;
9
+
use serde::de::DeserializeOwned;
10
+
11
+
/// `VecsAllowedQuery<QueryType>` is an extractor used to deserialize an
12
+
/// instance of `QueryType` from an HTTP request's query string. `QueryType`
13
+
/// is any structure of yours that implements [serde::Deserialize] and
14
+
/// [schemars::JsonSchema]. See the crate documentation for more information.
15
+
#[derive(Debug)]
16
+
pub struct VecsAllowedQuery<QueryType: DeserializeOwned + JsonSchema + Send + Sync> {
17
+
inner: QueryType,
18
+
}
19
+
impl<QueryType: DeserializeOwned + JsonSchema + Send + Sync> VecsAllowedQuery<QueryType> {
20
+
// TODO drop this in favor of Deref? + Display and Debug for convenience?
21
+
pub fn into_inner(self) -> QueryType {
22
+
self.inner
23
+
}
24
+
}
25
+
26
+
/// Given an HTTP request, pull out the query string and attempt to deserialize
27
+
/// it as an instance of `QueryType`.
28
+
fn http_request_load_query<QueryType>(
29
+
request: &RequestInfo,
30
+
) -> Result<VecsAllowedQuery<QueryType>, HttpError>
31
+
where
32
+
QueryType: DeserializeOwned + JsonSchema + Send + Sync,
33
+
{
34
+
let raw_query_string = request.uri().query().unwrap_or("");
35
+
// TODO-correctness: are query strings defined to be urlencoded in this way?
36
+
match serde_qs::from_str(raw_query_string) {
37
+
Ok(q) => Ok(VecsAllowedQuery { inner: q }),
38
+
Err(e) => Err(HttpError::for_bad_request(
39
+
None,
40
+
format!("unable to parse query string: {}", e),
41
+
)),
42
+
}
43
+
}
44
+
45
+
// The `SharedExtractor` implementation for Query<QueryType> describes how to
46
+
// construct an instance of `Query<QueryType>` from an HTTP request: namely, by
47
+
// parsing the query string to an instance of `QueryType`.
48
+
// TODO-cleanup We shouldn't have to use the "'static" bound on `QueryType`
49
+
// here. It seems like we ought to be able to use 'async_trait, but that
50
+
// doesn't seem to be defined.
51
+
#[async_trait]
52
+
impl<QueryType> SharedExtractor for VecsAllowedQuery<QueryType>
53
+
where
54
+
QueryType: JsonSchema + DeserializeOwned + Send + Sync + 'static,
55
+
{
56
+
async fn from_request<Context: ServerContext>(
57
+
rqctx: &RequestContext<Context>,
58
+
) -> Result<VecsAllowedQuery<QueryType>, HttpError> {
59
+
http_request_load_query(&rqctx.request)
60
+
}
61
+
62
+
fn metadata(body_content_type: ApiEndpointBodyContentType) -> ExtractorMetadata {
63
+
// HACK: would love to use Query here but it "helpfully" panics when it sees a Vec.
64
+
// we can't really get at enough of Query's logic to use it directly, sadly, so the
65
+
// resulting openapi docs suck (query params are listed as body payload, example
66
+
// codes make no sense, etc.)
67
+
//
68
+
// trying to hack the resulting ExtractorMetadata to look like Query's is a pain:
69
+
// things almost work out but then something in dropshot won't be `pub` and it falls
70
+
// apart. maybe it's possible, i didn't get it in the time i had.
71
+
dropshot::TypedBody::<QueryType>::metadata(body_content_type)
72
+
}
73
+
}
+43
-11
ufos/src/server.rs
+43
-11
ufos/src/server.rs
···
1
1
use crate::index_html::INDEX_HTML;
2
+
use crate::qs_query::VecsAllowedQuery;
2
3
use crate::storage::StoreReader;
3
4
use crate::store_types::{HourTruncatedCursor, WeekTruncatedCursor};
4
5
use crate::{ConsumerInfo, Cursor, JustCount, Nsid, NsidCount, OrderCollectionsBy, UFOsRecord};
···
16
17
use dropshot::Query;
17
18
use dropshot::RequestContext;
18
19
use dropshot::ServerBuilder;
20
+
19
21
use http::{Response, StatusCode};
20
22
use schemars::JsonSchema;
21
23
use serde::{Deserialize, Serialize};
···
83
85
storage: serde_json::Value,
84
86
consumer: ConsumerInfo,
85
87
}
86
-
/// Get meta information about UFOs itself
88
+
/// UFOs meta-info
87
89
#[endpoint {
88
90
method = GET,
89
91
path = "/meta"
···
145
147
}
146
148
}
147
149
}
148
-
/// Get recent records by collection
150
+
/// Record samples
151
+
///
152
+
/// Get most recent records seen in the firehose, by collection NSID
149
153
///
150
154
/// Multiple collections are supported. They will be delivered in one big array with no
151
155
/// specified order.
···
195
199
196
200
#[derive(Debug, Deserialize, JsonSchema)]
197
201
struct TotalSeenCollectionsQuery {
198
-
collection: String, // JsonSchema not implemented for Nsid :(
202
+
collection: Vec<String>, // JsonSchema not implemented for Nsid :(
203
+
/// Limit stats to those seen after this UTC datetime
204
+
///
205
+
/// default: 1 week ago
206
+
since: Option<DateTime<Utc>>,
207
+
/// Limit stats to those seen before this UTC datetime
208
+
///
209
+
/// default: now
210
+
until: Option<DateTime<Utc>>,
199
211
}
200
212
#[derive(Debug, Serialize, JsonSchema)]
201
213
struct TotalCounts {
202
214
total_creates: u64,
203
215
dids_estimate: u64,
204
216
}
205
-
/// Get total records seen by collection
217
+
/// Collection stats
218
+
///
219
+
/// Get stats for a collection over a specific time period
220
+
///
221
+
/// API docs note: the **Body** fields here are actually query parameters!!
222
+
///
223
+
/// Due to limitations with dropshot's query parsing (no support for sequences),
224
+
/// this is kind of the best i could do for now. sadly.
206
225
#[endpoint {
207
226
method = GET,
208
-
path = "/records/total-seen"
227
+
path = "/collections/stats"
209
228
}]
210
229
async fn get_records_total_seen(
211
230
ctx: RequestContext<Context>,
212
-
collection_query: Query<TotalSeenCollectionsQuery>,
231
+
query: VecsAllowedQuery<TotalSeenCollectionsQuery>,
213
232
) -> OkCorsResponse<HashMap<String, TotalCounts>> {
214
233
let Context { storage, .. } = ctx.context();
234
+
let q = query.into_inner();
215
235
216
-
let query = collection_query.into_inner();
217
-
let collections = to_multiple_nsids(&query.collection)
218
-
.map_err(|reason| HttpError::for_bad_request(None, reason))?;
236
+
log::warn!("collection: {:?}", q.collection);
237
+
238
+
let mut collections = Vec::with_capacity(q.collection.len());
239
+
for c in q.collection {
240
+
let Ok(nsid) = Nsid::new(c.clone()) else {
241
+
return Err(HttpError::for_bad_request(
242
+
None,
243
+
format!("could not parse collection to nsid: {c}"),
244
+
));
245
+
};
246
+
collections.push(nsid);
247
+
}
248
+
249
+
let since = q.since.map(dt_to_cursor).transpose()?;
250
+
let until = q.until.map(dt_to_cursor).transpose()?;
219
251
220
252
let mut seen_by_collection = HashMap::with_capacity(collections.len());
221
253
···
283
315
order: Option<CollectionsQueryOrder>,
284
316
}
285
317
286
-
/// Get collection with statistics
318
+
/// List collections (with stats)
287
319
///
288
320
/// ## To fetch a full list:
289
321
///
···
384
416
range: Vec<DateTime<Utc>>,
385
417
series: HashMap<String, Vec<JustCount>>,
386
418
}
387
-
/// Get timeseries data
419
+
/// Collection timeseries stats
388
420
#[endpoint {
389
421
method = GET,
390
422
path = "/timeseries"