Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
microcosm link aggregator is called constellation
+84
-47
+32
-32
Cargo.lock
+32
-32
Cargo.lock
···
494
494
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
495
495
496
496
[[package]]
497
+
name = "constellation"
498
+
version = "0.1.0"
499
+
dependencies = [
500
+
"anyhow",
501
+
"askama",
502
+
"axum",
503
+
"axum-extra",
504
+
"axum-metrics",
505
+
"bincode",
506
+
"clap",
507
+
"ctrlc",
508
+
"flume",
509
+
"fs4",
510
+
"headers-accept",
511
+
"links",
512
+
"mediatype",
513
+
"metrics",
514
+
"metrics-exporter-prometheus",
515
+
"metrics-process",
516
+
"rocksdb",
517
+
"serde",
518
+
"serde_with",
519
+
"tempfile",
520
+
"tinyjson",
521
+
"tokio",
522
+
"tokio-util",
523
+
"tower-http",
524
+
"tungstenite",
525
+
"zstd",
526
+
]
527
+
528
+
[[package]]
497
529
name = "core-foundation"
498
530
version = "0.9.4"
499
531
source = "registry+https://github.com/rust-lang/crates.io-index"
···
1130
1162
"cc",
1131
1163
"pkg-config",
1132
1164
"vcpkg",
1133
-
]
1134
-
1135
-
[[package]]
1136
-
name = "link_aggregator"
1137
-
version = "0.1.0"
1138
-
dependencies = [
1139
-
"anyhow",
1140
-
"askama",
1141
-
"axum",
1142
-
"axum-extra",
1143
-
"axum-metrics",
1144
-
"bincode",
1145
-
"clap",
1146
-
"ctrlc",
1147
-
"flume",
1148
-
"fs4",
1149
-
"headers-accept",
1150
-
"links",
1151
-
"mediatype",
1152
-
"metrics",
1153
-
"metrics-exporter-prometheus",
1154
-
"metrics-process",
1155
-
"rocksdb",
1156
-
"serde",
1157
-
"serde_with",
1158
-
"tempfile",
1159
-
"tinyjson",
1160
-
"tokio",
1161
-
"tokio-util",
1162
-
"tower-http",
1163
-
"tungstenite",
1164
-
"zstd",
1165
1165
]
1166
1166
1167
1167
[[package]]
+1
-1
Cargo.toml
+1
-1
Cargo.toml
+1
-1
link_aggregator/Cargo.toml
constellation/Cargo.toml
+1
-1
link_aggregator/Cargo.toml
constellation/Cargo.toml
+1
-1
link_aggregator/readme.md
constellation/readme.md
+1
-1
link_aggregator/readme.md
constellation/readme.md
link_aggregator/src/consumer/jetstream.rs
constellation/src/consumer/jetstream.rs
link_aggregator/src/consumer/jetstream.rs
constellation/src/consumer/jetstream.rs
link_aggregator/src/consumer/jsonl_file.rs
constellation/src/consumer/jsonl_file.rs
link_aggregator/src/consumer/jsonl_file.rs
constellation/src/consumer/jsonl_file.rs
+1
-1
link_aggregator/src/consumer/mod.rs
constellation/src/consumer/mod.rs
+1
-1
link_aggregator/src/consumer/mod.rs
constellation/src/consumer/mod.rs
···
3
3
4
4
use crate::storage::LinkStorage;
5
5
use anyhow::Result;
6
+
use constellation::{ActionableEvent, RecordId};
6
7
use jetstream::consume_jetstream;
7
8
use jsonl_file::consume_jsonl_file;
8
-
use link_aggregator::{ActionableEvent, RecordId};
9
9
use links::collect_links;
10
10
use metrics::{counter, describe_counter, describe_histogram, histogram, Unit};
11
11
use std::path::PathBuf;
link_aggregator/src/lib.rs
constellation/src/lib.rs
link_aggregator/src/lib.rs
constellation/src/lib.rs
link_aggregator/src/main.rs
constellation/src/main.rs
link_aggregator/src/main.rs
constellation/src/main.rs
link_aggregator/src/server/acceptable.rs
constellation/src/server/acceptable.rs
link_aggregator/src/server/acceptable.rs
constellation/src/server/acceptable.rs
link_aggregator/src/server/filters.rs
constellation/src/server/filters.rs
link_aggregator/src/server/filters.rs
constellation/src/server/filters.rs
+1
-1
link_aggregator/src/server/mod.rs
constellation/src/server/mod.rs
+1
-1
link_aggregator/src/server/mod.rs
constellation/src/server/mod.rs
+1
-1
link_aggregator/src/storage/mem_store.rs
constellation/src/storage/mem_store.rs
+1
-1
link_aggregator/src/storage/mem_store.rs
constellation/src/storage/mem_store.rs
···
1
1
use super::{LinkReader, LinkStorage, PagedAppendingCollection, StorageStats};
2
2
use anyhow::Result;
3
-
use link_aggregator::{ActionableEvent, Did, RecordId};
3
+
use constellation::{ActionableEvent, Did, RecordId};
4
4
use links::CollectedLink;
5
5
use std::collections::HashMap;
6
6
use std::sync::{Arc, Mutex};
+1
-1
link_aggregator/src/storage/mod.rs
constellation/src/storage/mod.rs
+1
-1
link_aggregator/src/storage/mod.rs
constellation/src/storage/mod.rs
+1
-1
link_aggregator/src/storage/rocks_store.rs
constellation/src/storage/rocks_store.rs
+1
-1
link_aggregator/src/storage/rocks_store.rs
constellation/src/storage/rocks_store.rs
···
1
1
use super::{ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, StorageStats};
2
2
use anyhow::{bail, Result};
3
3
use bincode::Options as BincodeOptions;
4
-
use link_aggregator::{Did, RecordId};
4
+
use constellation::{Did, RecordId};
5
5
use links::CollectedLink;
6
6
use rocksdb::{
7
7
AsColumnFamilyRef, ColumnFamilyDescriptor, DBWithThreadMode, IteratorMode, MergeOperands,
+1
-1
link_aggregator/templates/base.html.j2
constellation/templates/base.html.j2
+1
-1
link_aggregator/templates/base.html.j2
constellation/templates/base.html.j2
···
39
39
</style>
40
40
</head>
41
41
<body class="{% block body_classes %}{% endblock %}">
42
-
<h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/link_aggregator">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1>
42
+
<h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/constellation">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">µcosm</a>!</h1>
43
43
{% block content %}{% endblock %}
44
44
45
45
<footer>
link_aggregator/templates/hello.html.j2
constellation/templates/hello.html.j2
link_aggregator/templates/hello.html.j2
constellation/templates/hello.html.j2
link_aggregator/templates/links-all-count.html.j2
constellation/templates/links-all-count.html.j2
link_aggregator/templates/links-all-count.html.j2
constellation/templates/links-all-count.html.j2
link_aggregator/templates/links-count.html.j2
constellation/templates/links-count.html.j2
link_aggregator/templates/links-count.html.j2
constellation/templates/links-count.html.j2
link_aggregator/templates/links.html.j2
constellation/templates/links.html.j2
link_aggregator/templates/links.html.j2
constellation/templates/links.html.j2
link_aggregator/templates/try-it-macros.html.j2
constellation/templates/try-it-macros.html.j2
link_aggregator/templates/try-it-macros.html.j2
constellation/templates/try-it-macros.html.j2
link_aggregator/zstd/dictionary
constellation/zstd/dictionary
link_aggregator/zstd/dictionary
constellation/zstd/dictionary
+43
-6
readme.md
+43
-6
readme.md
···
1
-
µcosm links
2
-
===========
1
+
microcosm: links
2
+
================
3
+
4
+
this repo contains libraries and apps for working with cross-record references in at-protocol.
5
+
6
+
7
+
App: [Constellation](./tree/main/microcosm/)
8
+
--------------------------------------------
9
+
10
+
A global atproto backlink index ✨
11
+
12
+
- Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD
13
+
- Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs
14
+
- Simple JSON API
15
+
16
+
All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more.
17
+
18
+
- **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet.
19
+
- source: [./microcosm/](./tree/main/microcosm/)
20
+
- public instance: [links.bsky.bad-example.com](https://links.bsky.bad-example.com/)
21
+
22
+
_note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._
23
+
24
+
25
+
App: Spacedust
26
+
--------------
3
27
4
-
optimistically extract links from arbitrary atproto records, optionally resolving canonical representations and possibly validating StrongRefs.
28
+
A notification subscription service 💫
5
29
30
+
using the same "link source" concept as [constellation](./tree/main/microcosm), offer webhook notifications for new references created to records
6
31
7
-
status
8
-
------
32
+
- **status**: in design
9
33
10
-
not at all ready (yet)
34
+
35
+
Library: [links](./tree/main/links/)
36
+
------------------------------------
37
+
38
+
A rust crate (not published on crates.io yet) for optimistically parsing links out of arbitrary atproto PDS records, and potentially canonicalizing them
39
+
40
+
- **status**: unstable, might remain an internal lib for constellation (and spacedust, soon)
41
+
42
+
11
43
12
44
---
45
+
46
+
47
+
old notes follow, ignore
48
+
------------------------
49
+
13
50
14
51
as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document.
15
52