Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
add a "get all links" endpoint!
+94
-5
+39
-3
link_aggregator/readme.md
+39
-3
link_aggregator/readme.md
···
11
11
12
12
The number of backlinks to a URI from a specified collection + json path.
13
13
14
-
Required URL parameters
14
+
#### Required URL parameters
15
15
16
16
- `target` (required): the URI. must be URL-encoded.
17
17
- example: `at%3A%2F%2Fdid%3Aplc%3A57vlzz2egy6eqr4nksacmbht%2Fapp.bsky.feed.post%2F3lg2pgq3gq22b`
···
20
20
- `path` (required): the JSON path in referring documents to consider.
21
21
- example: `.subject.uri`
22
22
23
-
cURL Example: Get a count of all bluesky likes for a post
23
+
#### Response
24
+
25
+
A number (u64) in plain text format
26
+
27
+
#### cURL example: Get a count of all bluesky likes for a post
24
28
25
29
```bash
26
-
curl 'http://raspberrypi.local:6789/links/count?target=at%3A%2F%2Fdid%3Aplc%3A57vlzz2egy6eqr4nksacmbht%2Fapp.bsky.feed.post%2F3lg2pgq3gq22b&collection=app.bsky.feed.like&path=.subject.uri'
30
+
curl '<HOST>/links/count?target=at%3A%2F%2Fdid%3Aplc%3A57vlzz2egy6eqr4nksacmbht%2Fapp.bsky.feed.post%2F3lg2pgq3gq22b&collection=app.bsky.feed.like&path=.subject.uri'
27
31
28
32
40
33
+
```
34
+
35
+
### `GET /links/all/count`
36
+
37
+
The number of backlinks to a URI from any source collection or json path
38
+
39
+
#### Required URL parameters
40
+
41
+
- `target` (required): the URI. must be URL-encoded.
42
+
- example: `did:plc:vc7f4oafdgxsihk4cry2xpze`
43
+
44
+
#### Response
45
+
46
+
A JSON object `{[NSID]: {[JSON path]: [N]}}`
47
+
48
+
#### cURL example: Get reference counts to a DID from any collection at any path
49
+
50
+
```bash
51
+
curl '<HOST>/links/all/count?target=did:plc:vc7f4oafdgxsihk4cry2xpze'
52
+
53
+
curl '<HOST>/links/all/count?target=did:plc:vc7f4oafdgxsihk4cry2xpze'
54
+
{
55
+
"app.bsky.graph.block": { ".subject": 13 },
56
+
"app.bsky.graph.follow": { ".subject": 159 },
57
+
"app.bsky.feed.post": { ".facets[].features[].did": 16 },
58
+
"app.bsky.graph.listitem": { ".subject": 6 },
59
+
"app.bsky.graph.starterpack":
60
+
{
61
+
".feeds[].creator.did": 1,
62
+
".feeds[].creator.labels[].src": 1
63
+
}
64
+
}
29
65
```
30
66
31
67
+28
-2
link_aggregator/src/server.rs
+28
-2
link_aggregator/src/server.rs
···
1
-
use axum::{extract::Query, http, routing::get, Router};
1
+
use axum::{extract::Query, http, routing::get, Json, Router};
2
2
use serde::Deserialize;
3
+
use std::collections::HashMap;
3
4
use tokio::net::{TcpListener, ToSocketAddrs};
4
5
use tokio::sync::oneshot::Receiver;
5
6
use tokio::task::block_in_place;
···
15
16
.route("/", get(hello))
16
17
.route(
17
18
"/links/count",
18
-
get(move |query| async { block_in_place(|| count_links(query, store)) }),
19
+
get({
20
+
let store = store.clone();
21
+
move |query| async { block_in_place(|| count_links(query, store)) }
22
+
}),
23
+
)
24
+
.route(
25
+
"/links/all/count",
26
+
get({
27
+
let store = store.clone();
28
+
move |query| async { block_in_place(|| count_all_links(query, store)) }
29
+
}),
19
30
)
20
31
.layer(axum_metrics::MetricLayer::default());
21
32
···
50
61
.map(|c| c.to_string())
51
62
.map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)
52
63
}
64
+
65
+
#[derive(Deserialize)]
66
+
struct GetAllLinksQuery {
67
+
target: String,
68
+
}
69
+
type GetAllLinksResponse = HashMap<String, HashMap<String, u64>>;
70
+
fn count_all_links(
71
+
query: Query<GetAllLinksQuery>,
72
+
store: impl LinkReader,
73
+
) -> Result<Json<GetAllLinksResponse>, http::StatusCode> {
74
+
store
75
+
.get_all_counts(&query.target)
76
+
.map(Json)
77
+
.map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)
78
+
}
+4
link_aggregator/src/storage/mod.rs
+4
link_aggregator/src/storage/mod.rs
···
1
1
use anyhow::Result;
2
2
use link_aggregator::{ActionableEvent, Did, RecordId};
3
3
use links::CollectedLink;
4
+
use std::collections::HashMap;
4
5
5
6
pub mod mem_store;
6
7
pub use mem_store::MemStorage;
···
44
45
45
46
pub trait LinkReader: Clone + Send + Sync + 'static {
46
47
fn get_count(&self, target: &str, collection: &str, path: &str) -> Result<u64>;
48
+
fn get_all_counts(&self, _target: &str) -> Result<HashMap<String, HashMap<String, u64>>> {
49
+
Ok(HashMap::new())
50
+
}
47
51
48
52
// todo: remove it
49
53
fn summarize(&self, qsize: u32) {
+23
link_aggregator/src/storage/rocks_store.rs
+23
link_aggregator/src/storage/rocks_store.rs
···
371
371
let cf = self.db.cf_handle(LINK_TARGETS_CF).unwrap();
372
372
self.prefix_iter_cf(&cf, RecordLinkKeyDidIdPrefix(*did_id))
373
373
}
374
+
fn iter_targets_for_target(
375
+
&self,
376
+
target: &Target,
377
+
) -> impl Iterator<Item = (TargetKey, TargetId)> + use<'_> {
378
+
let cf = self.db.cf_handle(TARGET_IDS_CF).unwrap();
379
+
self.prefix_iter_cf(&cf, TargetIdTargetPrefix(target.clone()))
380
+
}
374
381
375
382
fn check_for_did_dups(&self, problem_did_id: &DidId) {
376
383
let cf = self.db.cf_handle(DID_IDS_CF).unwrap();
···
617
624
Ok(0)
618
625
}
619
626
}
627
+
fn get_all_counts(&self, target: &str) -> Result<HashMap<String, HashMap<String, u64>>> {
628
+
let mut out: HashMap<String, HashMap<String, u64>> = HashMap::new();
629
+
for (target_key, target_id) in self.iter_targets_for_target(&Target(target.into())) {
630
+
let TargetKey(_, Collection(ref collection), RPath(ref path)) = target_key;
631
+
let count = self.get_target_linkers(&target_id)?.count();
632
+
out.entry(collection.into())
633
+
.or_default()
634
+
.insert(path.clone(), count);
635
+
}
636
+
Ok(out)
637
+
}
620
638
}
621
639
622
640
trait AsRocksKey: Serialize {}
···
635
653
636
654
// target_ids table
637
655
impl AsRocksKey for &TargetKey {}
656
+
impl AsRocksKeyPrefix<TargetKey> for &TargetIdTargetPrefix {}
638
657
impl AsRocksValue for &TargetId {}
658
+
impl KeyFromRocks for TargetKey {}
639
659
impl ValueFromRocks for TargetId {}
640
660
641
661
// target_links table
···
734
754
// does this even work????
735
755
#[derive(Debug, Serialize, Deserialize)]
736
756
struct RecordLinkKeyDidIdPrefix(DidId);
757
+
758
+
#[derive(Debug, Serialize, Deserialize)]
759
+
struct TargetIdTargetPrefix(Target);
737
760
738
761
#[derive(Debug, Serialize, Deserialize)]
739
762
struct RecordLinkTarget(RPath, TargetId);