forked from
microcosm.blue/microcosm-rs
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
hello ufos
+113
-10
+1
-1
.github/workflows/checks.yml
+1
-1
.github/workflows/checks.yml
···
26
26
- name: get nightly toolchain for jetstream fmt
27
27
run: rustup toolchain install nightly --allow-downgrade -c rustfmt
28
28
- name: fmt
29
-
run: cargo fmt --package links --package constellation -- --check
29
+
run: cargo fmt --package links --package constellation --package ufos -- --check
30
30
- name: fmt jetstream (nightly)
31
31
run: cargo +nightly fmt --package jetstream -- --check
32
32
- name: clippy
+19
-8
Cargo.lock
+19
-8
Cargo.lock
···
105
105
106
106
[[package]]
107
107
name = "anyhow"
108
-
version = "1.0.95"
108
+
version = "1.0.97"
109
109
source = "registry+https://github.com/rust-lang/crates.io-index"
110
-
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
110
+
checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
111
111
112
112
[[package]]
113
113
name = "arrayvec"
···
521
521
522
522
[[package]]
523
523
name = "clap"
524
-
version = "4.5.28"
524
+
version = "4.5.31"
525
525
source = "registry+https://github.com/rust-lang/crates.io-index"
526
-
checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff"
526
+
checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767"
527
527
dependencies = [
528
528
"clap_builder",
529
529
"clap_derive",
···
531
531
532
532
[[package]]
533
533
name = "clap_builder"
534
-
version = "4.5.27"
534
+
version = "4.5.31"
535
535
source = "registry+https://github.com/rust-lang/crates.io-index"
536
-
checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7"
536
+
checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863"
537
537
dependencies = [
538
538
"anstream",
539
539
"anstyle",
···
2214
2214
2215
2215
[[package]]
2216
2216
name = "serde_json"
2217
-
version = "1.0.139"
2217
+
version = "1.0.140"
2218
2218
source = "registry+https://github.com/rust-lang/crates.io-index"
2219
-
checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6"
2219
+
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
2220
2220
dependencies = [
2221
2221
"itoa",
2222
2222
"memchr",
···
2672
2672
version = "1.17.0"
2673
2673
source = "registry+https://github.com/rust-lang/crates.io-index"
2674
2674
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
2675
+
2676
+
[[package]]
2677
+
name = "ufos"
2678
+
version = "0.1.0"
2679
+
dependencies = [
2680
+
"anyhow",
2681
+
"clap",
2682
+
"jetstream",
2683
+
"serde_json",
2684
+
"tokio",
2685
+
]
2675
2686
2676
2687
[[package]]
2677
2688
name = "unicase"
+1
-1
Makefile
+1
-1
Makefile
+23
jetstream/src/lib.rs
+23
jetstream/src/lib.rs
···
60
60
USWestTwo,
61
61
}
62
62
63
+
impl DefaultJetstreamEndpoints {
64
+
/// Helper to reference official jetstream instances by shortcut
65
+
///
66
+
/// This function will pass through a jetstream endpoint URL unless it matches a shortcut,
67
+
/// in which case it will be rewritten to the corresponding bluesky-operated jetstream endpoint
68
+
/// URL.
69
+
///
70
+
/// The shortcuts available are
71
+
/// - 'us-east-1'
72
+
/// - 'us-east-2'
73
+
/// - 'us-west-1'
74
+
/// - 'us-west-2'
75
+
pub fn endpoint_or_shortcut(s: &str) -> String {
76
+
match s {
77
+
"us-east-1" => DefaultJetstreamEndpoints::USEastOne.into(),
78
+
"us-east-2" => DefaultJetstreamEndpoints::USEastTwo.into(),
79
+
"us-west-1" => DefaultJetstreamEndpoints::USWestOne.into(),
80
+
"us-west-2" => DefaultJetstreamEndpoints::USWestTwo.into(),
81
+
custom => custom.into(),
82
+
}
83
+
}
84
+
}
85
+
63
86
impl From<DefaultJetstreamEndpoints> for String {
64
87
fn from(endpoint: DefaultJetstreamEndpoints) -> Self {
65
88
match endpoint {
+11
ufos/Cargo.toml
+11
ufos/Cargo.toml
···
1
+
[package]
2
+
name = "ufos"
3
+
version = "0.1.0"
4
+
edition = "2021"
5
+
6
+
[dependencies]
7
+
anyhow = "1.0.97"
8
+
clap = { version = "4.5.31", features = ["derive"] }
9
+
jetstream = { path = "../jetstream" }
10
+
serde_json = "1.0.140"
11
+
tokio = { version = "1.43.0", features = ["full", "sync", "time"] }
+57
ufos/src/main.rs
+57
ufos/src/main.rs
···
1
+
use clap::Parser;
2
+
use std::path::PathBuf;
3
+
use std::time::{Duration, Instant};
4
+
5
+
use jetstream::{
6
+
events::{commit::CommitEvent, JetstreamEvent::Commit},
7
+
DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector,
8
+
};
9
+
10
+
/// Aggregate links in the at-mosphere
11
+
#[derive(Parser, Debug)]
12
+
#[command(version, about, long_about = None)]
13
+
struct Args {
14
+
/// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value:
15
+
/// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2'
16
+
#[arg(long)]
17
+
jetstream: String,
18
+
/// Location to store persist data to disk
19
+
#[arg(long)]
20
+
data: PathBuf,
21
+
}
22
+
23
+
#[tokio::main]
24
+
async fn main() -> anyhow::Result<()> {
25
+
let args = Args::parse();
26
+
27
+
let config: JetstreamConfig<serde_json::Value> = JetstreamConfig {
28
+
endpoint: DefaultJetstreamEndpoints::endpoint_or_shortcut(&args.jetstream),
29
+
compression: JetstreamCompression::Zstd,
30
+
..Default::default()
31
+
};
32
+
33
+
let jetstream: JetstreamConnector<serde_json::Value> = JetstreamConnector::new(config)?;
34
+
let receiver = jetstream.connect().await?;
35
+
36
+
println!("Jetstream ready");
37
+
38
+
let print_throttle = Duration::from_millis(400);
39
+
let mut last = Instant::now();
40
+
while let Ok(event) = receiver.recv_async().await {
41
+
if let Commit(CommitEvent::Create { commit, .. }) = event {
42
+
let now = Instant::now();
43
+
let since = now - last;
44
+
if since >= print_throttle {
45
+
let overshoot = since - print_throttle; // adjust to keep the rate on average
46
+
last = now - overshoot;
47
+
println!(
48
+
"{}: {}",
49
+
&*commit.info.collection,
50
+
serde_json::to_string(&commit.record)?
51
+
);
52
+
}
53
+
}
54
+
}
55
+
56
+
Ok(())
57
+
}
ufos/src/serialize.rs
ufos/src/serialize.rs
This is a binary file and will not be displayed.