forked from
microcosm.blue/microcosm-rs
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
create html templates for browser exploration
+241
-39
+2
Cargo.lock
+2
Cargo.lock
+1
-1
link_aggregator/Cargo.toml
+1
-1
link_aggregator/Cargo.toml
+1
-1
link_aggregator/src/lib.rs
+1
-1
link_aggregator/src/lib.rs
+19
link_aggregator/src/server/filters.rs
+19
link_aggregator/src/server/filters.rs
···
1
+
use links::{parse_any_link, Link};
2
+
3
+
pub fn to_browseable(s: &str) -> askama::Result<Option<String>> {
4
+
Ok({
5
+
if let Some(link) = parse_any_link(s) {
6
+
match link {
7
+
Link::AtUri(at_uri) => at_uri.strip_prefix("at://").map(|noproto| {
8
+
format!("https://atproto-browser-plus-links.vercel.app/at/{noproto}")
9
+
}),
10
+
Link::Did(did) => Some(format!(
11
+
"https://atproto-browser-plus-links.vercel.app/at/{did}"
12
+
)),
13
+
Link::Uri(uri) => Some(uri),
14
+
}
15
+
} else {
16
+
None
17
+
}
18
+
})
19
+
}
+67
-32
link_aggregator/src/server/mod.rs
+67
-32
link_aggregator/src/server/mod.rs
···
1
1
use askama::Template;
2
-
use axum::{
3
-
extract::Query,
4
-
http,
5
-
response::{IntoResponse, Json},
6
-
routing::get,
7
-
Router,
8
-
};
2
+
use axum::{extract::Query, http, response::IntoResponse, routing::get, Router};
9
3
use bincode::Options;
10
4
use serde::{Deserialize, Serialize};
11
5
use serde_with::serde_as;
···
18
12
use link_aggregator::RecordId;
19
13
20
14
mod acceptable;
15
+
mod filters;
21
16
22
17
use acceptable::{acceptable, ExtractAccept};
23
18
···
35
30
"/links/count",
36
31
get({
37
32
let store = store.clone();
38
-
move |query| async { block_in_place(|| count_links(query, store)) }
33
+
move |accept, query| async { block_in_place(|| count_links(accept, query, store)) }
39
34
}),
40
35
)
41
36
.route(
42
37
"/links",
43
38
get({
44
39
let store = store.clone();
45
-
move |query| async { block_in_place(|| get_links(query, store)) }
40
+
move |accept, query| async { block_in_place(|| get_links(accept, query, store)) }
46
41
}),
47
42
)
48
43
.route(
49
44
"/links/all/count",
50
45
get({
51
46
let store = store.clone();
52
-
move |query| async { block_in_place(|| count_all_links(query, store)) }
47
+
move |accept, query| async {
48
+
block_in_place(|| count_all_links(accept, query, store))
49
+
}
53
50
}),
54
51
)
55
52
.layer(axum_metrics::MetricLayer::default());
···
65
62
66
63
#[derive(Template, Serialize, Deserialize)]
67
64
#[template(path = "hello.html.j2")]
68
-
struct HelloReponse {}
65
+
struct HelloReponse {
66
+
help: &'static str,
67
+
}
69
68
async fn hello(accept: ExtractAccept) -> impl IntoResponse {
70
-
let o = HelloReponse {};
71
-
acceptable(accept, o)
69
+
acceptable(accept, HelloReponse {
70
+
help: "open this URL in a web browser (or request with Accept: text/html) for information about this API."
71
+
})
72
72
}
73
73
74
-
#[derive(Deserialize)]
74
+
#[derive(Clone, Deserialize)]
75
75
struct GetLinksCountQuery {
76
76
target: String,
77
77
collection: String,
78
78
path: String,
79
79
}
80
+
#[derive(Template, Serialize)]
81
+
#[template(path = "links-count.html.j2")]
82
+
struct GetLinksCountResponse {
83
+
total: u64,
84
+
#[serde(skip_serializing)]
85
+
query: GetLinksCountQuery,
86
+
}
80
87
fn count_links(
88
+
accept: ExtractAccept,
81
89
query: Query<GetLinksCountQuery>,
82
90
store: impl LinkReader,
83
-
) -> Result<String, http::StatusCode> {
84
-
store
91
+
) -> Result<impl IntoResponse, http::StatusCode> {
92
+
let total = store
85
93
.get_count(&query.target, &query.collection, &query.path)
86
-
.map(|c| c.to_string())
87
-
.map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)
94
+
.map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?;
95
+
Ok(acceptable(
96
+
accept,
97
+
GetLinksCountResponse {
98
+
total,
99
+
query: (*query).clone(),
100
+
},
101
+
))
88
102
}
89
103
90
-
#[derive(Deserialize)]
104
+
#[derive(Clone, Deserialize)]
91
105
struct GetLinkItemsQuery {
92
106
target: String,
93
107
collection: String,
···
96
110
limit: Option<u64>,
97
111
// TODO: allow reverse (er, forward) order as well
98
112
}
99
-
#[derive(Serialize)]
113
+
#[derive(Template, Serialize)]
114
+
#[template(path = "links.html.j2")]
100
115
struct GetLinkItemsResponse {
101
116
// what does staleness mean?
102
117
// - new links have appeared. would be nice to offer a `since` cursor to fetch these. and/or,
···
104
119
total: u64,
105
120
linking_records: Vec<RecordId>,
106
121
cursor: Option<OpaqueApiCursor>,
122
+
#[serde(skip_serializing)]
123
+
query: GetLinkItemsQuery,
107
124
}
108
125
fn get_links(
126
+
accept: ExtractAccept,
109
127
query: Query<GetLinkItemsQuery>,
110
128
store: impl LinkReader,
111
-
) -> Result<Json<GetLinkItemsResponse>, http::StatusCode> {
129
+
) -> Result<impl IntoResponse, http::StatusCode> {
112
130
let until = query
113
131
.cursor
114
132
.clone()
···
133
151
.into()
134
152
});
135
153
136
-
Ok(Json(GetLinkItemsResponse {
137
-
total: paged.version.0,
138
-
linking_records: paged.items,
139
-
cursor,
140
-
}))
154
+
Ok(acceptable(
155
+
accept,
156
+
GetLinkItemsResponse {
157
+
total: paged.version.0,
158
+
linking_records: paged.items,
159
+
cursor,
160
+
query: (*query).clone(),
161
+
},
162
+
))
141
163
}
142
164
143
-
#[derive(Deserialize)]
165
+
#[derive(Clone, Deserialize)]
144
166
struct GetAllLinksQuery {
145
167
target: String,
146
168
}
147
-
type GetAllLinksResponse = HashMap<String, HashMap<String, u64>>;
169
+
#[derive(Template, Serialize)]
170
+
#[template(path = "links-all-count.html.j2")]
171
+
struct GetAllLinksResponse {
172
+
links: HashMap<String, HashMap<String, u64>>,
173
+
#[serde(skip_serializing)]
174
+
query: GetAllLinksQuery,
175
+
}
148
176
fn count_all_links(
177
+
accept: ExtractAccept,
149
178
query: Query<GetAllLinksQuery>,
150
179
store: impl LinkReader,
151
-
) -> Result<Json<GetAllLinksResponse>, http::StatusCode> {
152
-
store
180
+
) -> Result<impl IntoResponse, http::StatusCode> {
181
+
let links = store
153
182
.get_all_counts(&query.target)
154
-
.map(Json)
155
-
.map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)
183
+
.map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?;
184
+
Ok(acceptable(
185
+
accept,
186
+
GetAllLinksResponse {
187
+
links,
188
+
query: (*query).clone(),
189
+
},
190
+
))
156
191
}
157
192
158
193
#[serde_as]
+6
link_aggregator/src/storage/rocks_store.rs
+6
link_aggregator/src/storage/rocks_store.rs
···
753
753
let mut items = Vec::with_capacity(did_id_rkeys.len());
754
754
// TODO: use get-many (or multi-get or whatever it's called)
755
755
for (did_id, rkey) in did_id_rkeys {
756
+
if did_id.is_empty() {
757
+
continue;
758
+
}
756
759
if let Some(did) = self.did_id_table.get_val_from_id(&self.db, did_id.0)? {
757
760
let Some(DidIdValue(_, active)) = self.did_id_table.get_id_val(&self.db, &did)?
758
761
else {
···
867
870
impl DidId {
868
871
fn empty() -> Self {
869
872
DidId(0)
873
+
}
874
+
fn is_empty(&self) -> bool {
875
+
self.0 == 0
870
876
}
871
877
}
872
878
+27
-3
link_aggregator/templates/base.html.j2
+27
-3
link_aggregator/templates/base.html.j2
···
8
8
body {
9
9
font-family: sans-serif;
10
10
}
11
-
code {
11
+
h1 {
12
+
font-size: 1em;
13
+
}
14
+
h2, h3 {
15
+
margin-top: 2.4em;
16
+
}
17
+
code, .code {
12
18
background: #def;
13
19
display: inline-block;
14
20
font-size: 1em;
15
21
padding: 0 0.3em;
16
22
border-radius: 0.2em;
17
23
}
18
-
h2, h3 {
19
-
margin-top: 2.4em;
24
+
pre {
25
+
box-sizing: border-box;
26
+
margin-bottom: 0;
27
+
}
28
+
pre.code {
29
+
padding: 0.5em 0.3em;
30
+
max-width: 100%;
31
+
}
32
+
details {
33
+
margin: 2em 0 3em;
34
+
}
35
+
footer {
36
+
margin-top: 5em;
37
+
font-size: 0.8em;
20
38
}
21
39
</style>
22
40
</head>
23
41
<body>
42
+
<h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/link_aggregator">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1>
24
43
{% block content %}{% endblock %}
44
+
45
+
<footer>
46
+
<p>To get this API to return JSON, set request header <code>Accept: application/json</code>.</p>
47
+
<p><a href="/">API docs main</a></p>
48
+
</footer>
25
49
</body>
26
50
</html>
-2
link_aggregator/templates/hello.html.j2
-2
link_aggregator/templates/hello.html.j2
···
3
3
{% block title %}Hello!{% endblock %}
4
4
5
5
{% block content %}
6
-
<h1>This is an <a href="https://github.com/at-ucosm/links/tree/main/link_aggregator">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1>
7
-
8
6
<p>Every interaction in Bluesky and atproto at large tends to appear as a <em>link</em> from a new repository record to <em>somewhere</em>: liking a post creates a record with a link to the post, blocking a spammer creates a record with reference to their DID.</p>
9
7
10
8
<p>This service attempts to aggregate all of these links, globally, from all content coming throught the firehose. It provides generic API endpoints to answer questions like <strong>how many likes does a post have</strong> and <strong>who follows a user</strong>.</p>
+34
link_aggregator/templates/links-all-count.html.j2
+34
link_aggregator/templates/links-all-count.html.j2
···
1
+
{% extends "base.html.j2" %}
2
+
3
+
{% block title %}All link counts{% endblock %}
4
+
5
+
{% block content %}
6
+
7
+
<h2>
8
+
All links to <code>{{ query.target }}</code>
9
+
{% if let Some(browseable_uri) = query.target|to_browseable %}
10
+
<small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small>
11
+
{% endif %}
12
+
</h2>
13
+
14
+
<form method="get" action="/links/all/count">
15
+
<pre class="code"><strong>GET</strong> /links?target=<input type="text" name="target" value="{{ query.target }}" placeholder="target" /> <button type="submit">get link counts</button></pre>
16
+
</form>
17
+
18
+
<h3>Links by collection and path:</h3>
19
+
20
+
<pre style="display: block; margin: 1em 2em" class="code">
21
+
{%- for (collection, collection_links) in links -%}
22
+
<strong>{{ collection }}</strong>
23
+
{%- for (path, count) in collection_links %}
24
+
{{ path }}: <a href="/links?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ count }} links</a></li>
25
+
{%- endfor %}
26
+
27
+
{% endfor -%}
28
+
</pre>
29
+
<details>
30
+
<summary>Raw JSON response</summary>
31
+
<pre class="code">{{ self|tojson }}</pre>
32
+
</details>
33
+
34
+
{% endblock %}
+32
link_aggregator/templates/links-count.html.j2
+32
link_aggregator/templates/links-count.html.j2
···
1
+
{% extends "base.html.j2" %}
2
+
3
+
{% block title %}Link count{% endblock %}
4
+
5
+
{% block content %}
6
+
7
+
<h2>
8
+
Total links to <code>{{ query.target }}</code>
9
+
{% if let Some(browseable_uri) = query.target|to_browseable %}
10
+
<small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small>
11
+
{% endif %}
12
+
</h2>
13
+
14
+
15
+
<p><strong><code>{{ total }}</code></strong> total links from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p>
16
+
17
+
<form method="get" action="/links/count">
18
+
<pre class="code"><strong>GET</strong> /links/count
19
+
?target= <input type="text" name="target" value="{{ query.target }}" placeholder="target" />
20
+
&collection= <input type="text" name="collection" value="{{ query.collection }}" placeholder="collection" />
21
+
&path= <input type="text" name="path" value="{{ query.path }}" placeholder="path" />
22
+
<button type="submit">get links count</button></pre>
23
+
</form>
24
+
25
+
<p>See these links at <code>/links</code>: <a href="/links?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode() }}">/links?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}</a></p>
26
+
27
+
<details>
28
+
<summary>Raw JSON response</summary>
29
+
<pre class="code">{{ self|tojson }}</pre>
30
+
</details>
31
+
32
+
{% endblock %}
+52
link_aggregator/templates/links.html.j2
+52
link_aggregator/templates/links.html.j2
···
1
+
{% extends "base.html.j2" %}
2
+
3
+
{% block title %}Links{% endblock %}
4
+
5
+
{% block content %}
6
+
7
+
<h2>
8
+
Links to <code>{{ query.target }}</code>
9
+
{% if let Some(browseable_uri) = query.target|to_browseable %}
10
+
<small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small>
11
+
{% endif %}
12
+
</h2>
13
+
14
+
<p><strong>{{ total }} links</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p>
15
+
16
+
<form method="get" action="/links">
17
+
<pre class="code"><strong>GET</strong> /links
18
+
?target= <input type="text" name="target" value="{{ query.target }}" placeholder="target" />
19
+
&collection= <input type="text" name="collection" value="{{ query.collection }}" placeholder="collection" />
20
+
&path= <input type="text" name="path" value="{{ query.path }}" placeholder="path" />
21
+
<button type="submit">get links</button></pre>
22
+
</form>
23
+
24
+
<p>See all links to this target <code>/links/all/count</code>: <a href="/links/all/count?target={{ query.target|urlencode }}">/links/all/count?target={{ query.target }}</a></p>
25
+
26
+
<h3>Links, most recent first:</h3>
27
+
28
+
{% for record in linking_records %}
29
+
<pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ record.did().0 }} (<a href="/links/all/count?target={{ record.did().0|urlencode }}">DID links</a>)
30
+
<strong>Collection</strong>: {{ record.collection }}
31
+
<strong>RKey</strong>: {{ record.rkey }}
32
+
-> <a href="https://atproto-browser-plus-links.vercel.app/at/{{ record.did().0 }}/{{ record.collection }}/{{ record.rkey }}">browse record</a></pre>
33
+
{% endfor %}
34
+
35
+
{% if let Some(c) = cursor %}
36
+
<form method="get" action="/links">
37
+
<input type="hidden" name="target" value="{{ query.target }}" />
38
+
<input type="hidden" name="collection" value="{{ query.collection }}" />
39
+
<input type="hidden" name="path" value="{{ query.path }}" />
40
+
<input type="hidden" name="cursor" value={{ c|json|safe }} />
41
+
<button type="submit">next page…</button>
42
+
</form>
43
+
{% else %}
44
+
<button disabled><em>end of results</em></button>
45
+
{% endif %}
46
+
47
+
<details>
48
+
<summary>Raw JSON response</summary>
49
+
<pre class="code">{{ self|tojson }}</pre>
50
+
</details>
51
+
52
+
{% endblock %}