this repo has no description
tidy up some indexer logic
- remove sqlx cache bc i aint usin it rn (will add it back l8r for docker build)
- document how im going to do backfill/etc
- add meta table with a rev column. this is to store repo rev for diffing/etc
+29
-39
-12
.sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json
-12
.sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json
···
1
-
{
2
-
"db_name": "PostgreSQL",
3
-
"query": "CREATE TABLE IF NOT EXISTS foreign_records (\n did TEXT,\n collection TEXT,\n rkey TEXT,\n record JSON NOT NULL,\n PRIMARY KEY (did, collection, rkey)\n );",
4
-
"describe": {
5
-
"columns": [],
6
-
"parameters": {
7
-
"Left": []
8
-
},
9
-
"nullable": []
10
-
},
11
-
"hash": "214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca"
12
-
}
-12
.sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json
-12
.sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json
···
1
-
{
2
-
"db_name": "PostgreSQL",
3
-
"query": "CREATE TABLE IF NOT EXISTS blobs (\n did TEXT,\n cid TEXT,\n blob bytea NOT NULL,\n PRIMARY KEY (did, cid)\n )",
4
-
"describe": {
5
-
"columns": [],
6
-
"parameters": {
7
-
"Left": []
8
-
},
9
-
"nullable": []
10
-
},
11
-
"hash": "2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b"
12
-
}
-12
.sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json
-12
.sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json
···
1
-
{
2
-
"db_name": "PostgreSQL",
3
-
"query": "CREATE TABLE IF NOT EXISTS records (\n collection TEXT,\n rkey TEXT,\n record JSON NOT NULL,\n PRIMARY KEY (collection, rkey)\n );",
4
-
"describe": {
5
-
"columns": [],
6
-
"parameters": {
7
-
"Left": []
8
-
},
9
-
"nullable": []
10
-
},
11
-
"hash": "94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2"
12
-
}
-2
sqlx.sh
-2
sqlx.sh
+15
-1
src/db.rs
+15
-1
src/db.rs
···
58
58
panic!();
59
59
};
60
60
61
-
return conn;
61
+
if let Err(err) = query!(
62
+
"CREATE TABLE IF NOT EXISTS meta (
63
+
did TEXT,
64
+
rev TEXT,
65
+
PRIMARY KEY (did)
66
+
);"
67
+
)
68
+
.execute(&conn)
69
+
.await
70
+
{
71
+
println!("Creating table `meta`: \n{err}");
72
+
panic!();
73
+
};
74
+
75
+
conn
62
76
}
+14
src/main.rs
+14
src/main.rs
···
2
2
mod db;
3
3
mod resolver;
4
4
5
+
/// backfill works as follows (https://docs.bsky.app/docs/advanced-guides/backfill)
6
+
///
7
+
/// 1. resolve did -> pds
8
+
/// 2. stream com.atproto.sync.subscribeRepos to a buffer
9
+
/// 3. get a car file from com.atproto.sync.getRepo (diff if a rev is stored in database)
10
+
/// 4. apply car file diff to database (incl rev)
11
+
/// 5. start playing events from buffer
12
+
/// a. drop all events from other users
13
+
/// b. drop all events with a lower rev than current rev
14
+
/// c. apply event & update rev
15
+
/// d. (non blocking) get blobs if missing
16
+
/// e. (non blocking) parse for strongref and store strongrefs
17
+
/// f. (non blocking) trigger garbage collection of blobs and strongref
18
+
/// 6. once buffer is empty, parse events live
5
19
#[tokio::main]
6
20
async fn main() {
7
21
println!("User: {}", *config::USER);