microcosm.blue / microcosm-rs
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
75
fork

Configure Feed

Select the types of activity you want to include in your feed.

GET /links/all with distinct did counts

phil c44ab956 db35300a

+217 -30
+4 -4
constellation/readme.md
··· 136 136 - [x] either count or estimate the total number of links added (distinct from link targets) 137 137 - [x] jetstream: don't crash on connection refused (retry * backoff) 138 138 - [x] allow cors requests (ie. atproto-browser. (but it's really meant for backends)) 139 - - [~] api: get distinct linking dids (https://bsky.app/profile/bnewbold.net/post/3lhhzejv7zc2h) 140 - - [~] endpoint for count 141 - - [~] endpoint for listing them 142 - - [ ] add to exploratory /all endpoint 139 + - [x] api: get distinct linking dids (https://bsky.app/profile/bnewbold.net/post/3lhhzejv7zc2h) 140 + - [x] endpoint for count 141 + - [x] endpoint for listing them 142 + - [x] add to exploratory /all endpoint 143 143 144 144 cache 145 145 - [ ] set api response headers
+7
constellation/src/lib.rs
··· 45 45 self.rkey.clone() 46 46 } 47 47 } 48 + 49 + /// maybe the worst type in this repo, and there are some bad types 50 + #[derive(Debug, Serialize, PartialEq)] 51 + pub struct CountsByCount { 52 + pub records: u64, 53 + pub distinct_dids: u64, 54 + }
+40 -2
constellation/src/server/mod.rs
··· 9 9 use tokio_util::sync::CancellationToken; 10 10 11 11 use crate::storage::LinkReader; 12 - use constellation::{Did, RecordId}; 12 + use constellation::{CountsByCount, Did, RecordId}; 13 13 14 14 mod acceptable; 15 15 mod filters; ··· 59 59 }), 60 60 ) 61 61 .route( 62 + // deprecated 62 63 "/links/all/count", 63 64 get({ 64 65 let store = store.clone(); 65 66 move |accept, query| async { 66 67 block_in_place(|| count_all_links(accept, query, store)) 68 + } 69 + }), 70 + ) 71 + .route( 72 + "/links/all", 73 + get({ 74 + let store = store.clone(); 75 + move |accept, query| async { 76 + block_in_place(|| explore_links(accept, query, store)) 67 77 } 68 78 }), 69 79 ) ··· 289 299 store: impl LinkReader, 290 300 ) -> Result<impl IntoResponse, http::StatusCode> { 291 301 let links = store 292 - .get_all_counts(&query.target) 302 + .get_all_record_counts(&query.target) 293 303 .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 294 304 Ok(acceptable( 295 305 accept, 296 306 GetAllLinksResponse { 307 + links, 308 + query: (*query).clone(), 309 + }, 310 + )) 311 + } 312 + 313 + #[derive(Clone, Deserialize)] 314 + struct ExploreLinksQuery { 315 + target: String, 316 + } 317 + #[derive(Template, Serialize)] 318 + #[template(path = "explore-links.html.j2")] 319 + struct ExploreLinksResponse { 320 + links: HashMap<String, HashMap<String, CountsByCount>>, 321 + #[serde(skip_serializing)] 322 + query: ExploreLinksQuery, 323 + } 324 + fn explore_links( 325 + accept: ExtractAccept, 326 + query: Query<ExploreLinksQuery>, 327 + store: impl LinkReader, 328 + ) -> Result<impl IntoResponse, http::StatusCode> { 329 + let links = store 330 + .get_all_counts(&query.target) 331 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 332 + Ok(acceptable( 333 + accept, 334 + ExploreLinksResponse { 297 335 links, 298 336 query: (*query).clone(), 299 337 },
+29 -2
constellation/src/storage/mem_store.rs
··· 1 1 use super::{LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 2 use anyhow::Result; 3 - use constellation::{ActionableEvent, Did, RecordId}; 3 + use constellation::{ActionableEvent, CountsByCount, Did, RecordId}; 4 4 use links::CollectedLink; 5 5 use std::collections::{HashMap, HashSet}; 6 6 use std::sync::{Arc, Mutex}; ··· 272 272 }) 273 273 } 274 274 275 - fn get_all_counts(&self, target: &str) -> Result<HashMap<String, HashMap<String, u64>>> { 275 + fn get_all_record_counts(&self, target: &str) -> Result<HashMap<String, HashMap<String, u64>>> { 276 276 let data = self.0.lock().unwrap(); 277 277 let mut out: HashMap<String, HashMap<String, u64>> = HashMap::new(); 278 278 if let Some(asdf) = data.targets.get(&Target::new(target)) { ··· 281 281 out.entry(collection.to_string()) 282 282 .or_default() 283 283 .insert(path.to_string(), count); 284 + } 285 + } 286 + Ok(out) 287 + } 288 + 289 + fn get_all_counts( 290 + &self, 291 + target: &str, 292 + ) -> Result<HashMap<String, HashMap<String, CountsByCount>>> { 293 + let data = self.0.lock().unwrap(); 294 + let mut out: HashMap<String, HashMap<String, CountsByCount>> = HashMap::new(); 295 + if let Some(asdf) = data.targets.get(&Target::new(target)) { 296 + for (Source { collection, path }, linkers) in asdf { 297 + let records = linkers.iter().flatten().count() as u64; 298 + let distinct_dids = linkers 299 + .iter() 300 + .flatten() 301 + .map(|(did, _)| did) 302 + .collect::<HashSet<_>>() 303 + .len() as u64; 304 + out.entry(collection.to_string()).or_default().insert( 305 + path.to_string(), 306 + CountsByCount { 307 + records, 308 + distinct_dids, 309 + }, 310 + ); 284 311 } 285 312 } 286 313 Ok(out)
+33 -3
constellation/src/storage/mod.rs
··· 1 1 use anyhow::Result; 2 - use constellation::{ActionableEvent, Did, RecordId}; 2 + use constellation::{ActionableEvent, CountsByCount, Did, RecordId}; 3 3 use std::collections::HashMap; 4 4 5 5 pub mod mem_store; ··· 68 68 until: Option<u64>, 69 69 ) -> Result<PagedAppendingCollection<Did>>; // TODO: reflect dedups in cursor 70 70 71 - fn get_all_counts(&self, _target: &str) -> Result<HashMap<String, HashMap<String, u64>>>; 71 + fn get_all_record_counts(&self, _target: &str) 72 + -> Result<HashMap<String, HashMap<String, u64>>>; 73 + 74 + fn get_all_counts( 75 + &self, 76 + _target: &str, 77 + ) -> Result<HashMap<String, HashMap<String, CountsByCount>>>; 72 78 73 79 /// assume all stats are estimates, since exact counts are very challenging for LSMs 74 80 fn get_stats(&self) -> Result<StorageStats>; ··· 153 159 } 154 160 ); 155 161 assert_eq!(storage.get_all_counts("bad-example.com")?, HashMap::new()); 162 + assert_eq!( 163 + storage.get_all_record_counts("bad-example.com")?, 164 + HashMap::new() 165 + ); 156 166 157 167 assert_stats(storage.get_stats()?, 0..=0, 0..=0, 0..=0); 158 168 }); ··· 1022 1032 }, 1023 1033 0, 1024 1034 )?; 1025 - assert_eq!(storage.get_all_counts("a.com")?, { 1035 + assert_eq!(storage.get_all_record_counts("a.com")?, { 1026 1036 let mut counts = HashMap::new(); 1027 1037 let mut t_c_counts = HashMap::new(); 1028 1038 t_c_counts.insert(".abc.uri".into(), 1); 1029 1039 t_c_counts.insert(".def.uri".into(), 1); 1040 + counts.insert("app.t.c".into(), t_c_counts); 1041 + counts 1042 + }); 1043 + assert_eq!(storage.get_all_counts("a.com")?, { 1044 + let mut counts = HashMap::new(); 1045 + let mut t_c_counts = HashMap::new(); 1046 + t_c_counts.insert( 1047 + ".abc.uri".into(), 1048 + CountsByCount { 1049 + records: 1, 1050 + distinct_dids: 1, 1051 + }, 1052 + ); 1053 + t_c_counts.insert( 1054 + ".def.uri".into(), 1055 + CountsByCount { 1056 + records: 1, 1057 + distinct_dids: 1, 1058 + }, 1059 + ); 1030 1060 counts.insert("app.t.c".into(), t_c_counts); 1031 1061 counts 1032 1062 });
+31 -8
constellation/src/storage/rocks_store.rs
··· 1 1 use super::{ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; 2 2 use anyhow::{bail, Result}; 3 3 use bincode::Options as BincodeOptions; 4 - use constellation::{Did, RecordId}; 4 + use constellation::{CountsByCount, Did, RecordId}; 5 5 use links::CollectedLink; 6 6 use rocksdb::{ 7 7 AsColumnFamilyRef, ColumnFamilyDescriptor, DBWithThreadMode, IteratorMode, MergeOperands, ··· 654 654 RPath(path.to_string()), 655 655 ); 656 656 if let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? { 657 - let TargetLinkers(alives) = self.get_target_linkers(&target_id)?; 658 - Ok(alives // TODO: maybe make this a method on TargetLinkers? 659 - .iter() 660 - .filter_map(|(DidId(id), _)| if *id == 0 { None } else { Some(id) }) 661 - .collect::<HashSet<_>>() 662 - .len() as u64) 657 + Ok(self.get_target_linkers(&target_id)?.count_distinct_dids()) 663 658 } else { 664 659 Ok(0) 665 660 } ··· 789 784 }) 790 785 } 791 786 792 - fn get_all_counts(&self, target: &str) -> Result<HashMap<String, HashMap<String, u64>>> { 787 + fn get_all_record_counts(&self, target: &str) -> Result<HashMap<String, HashMap<String, u64>>> { 793 788 let mut out: HashMap<String, HashMap<String, u64>> = HashMap::new(); 794 789 for (target_key, target_id) in self.iter_targets_for_target(&Target(target.into())) { 795 790 let TargetKey(_, Collection(ref collection), RPath(ref path)) = target_key; ··· 801 796 Ok(out) 802 797 } 803 798 799 + fn get_all_counts( 800 + &self, 801 + target: &str, 802 + ) -> Result<HashMap<String, HashMap<String, CountsByCount>>> { 803 + let mut out: HashMap<String, HashMap<String, CountsByCount>> = HashMap::new(); 804 + for (target_key, target_id) in self.iter_targets_for_target(&Target(target.into())) { 805 + let TargetKey(_, Collection(ref collection), RPath(ref path)) = target_key; 806 + let target_linkers = self.get_target_linkers(&target_id)?; 807 + let (records, _) = target_linkers.count(); 808 + let distinct_dids = target_linkers.count_distinct_dids(); 809 + out.entry(collection.into()).or_default().insert( 810 + path.clone(), 811 + CountsByCount { 812 + records, 813 + distinct_dids, 814 + }, 815 + ); 816 + } 817 + Ok(out) 818 + } 819 + 804 820 fn get_stats(&self) -> Result<StorageStats> { 805 821 let dids = self.did_id_table.estimate_count(); 806 822 let targetables = self.target_id_table.estimate_count(); ··· 939 955 let alive = self.0.iter().filter(|(DidId(id), _)| *id != 0).count() as u64; 940 956 let gone = total - alive; 941 957 (alive, gone) 958 + } 959 + fn count_distinct_dids(&self) -> u64 { 960 + self.0 961 + .iter() 962 + .filter_map(|(DidId(id), _)| if *id == 0 { None } else { Some(id) }) 963 + .collect::<HashSet<_>>() 964 + .len() as u64 942 965 } 943 966 } 944 967
+3 -3
constellation/templates/base.html.j2
··· 49 49 </style> 50 50 </head> 51 51 <body class="{% block body_classes %}{% endblock %}"> 52 - <h1><a href="/">This</a> is an <a href="https://github.com/at-ucosm/links/tree/main/constellation">atproto link aggregator</a> server from <a href="https://github.com/at-ucosm">microcosm</a>!</h1> 52 + <h1><a href="/">This</a> is a <a href="https://github.com/at-ucosm/links/tree/main/constellation">constellation 🌌</a> server from <a href="https://github.com/at-ucosm">microcosm</a> ✨</h1> 53 53 {% block content %}{% endblock %} 54 54 55 55 <footer> 56 - <p>To get this API to return JSON, set request header <code>Accept: application/json</code>.</p> 57 - <p><a href="/">API docs main</a></p> 56 + <p>To get this response as JSON, set request header <code>Accept: application/json</code>.</p> 57 + <p><a href="/">Constellation API docs main</a></p> 58 58 </footer> 59 59 </body> 60 60 </html>
+3 -3
constellation/templates/dids.html.j2
··· 17 17 <p><strong>{{ total }} dids</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p> 18 18 19 19 <ul> 20 - <li>See linking records to this target <code>/links</code>: <a href="/links?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li> 21 - <li>See all links to this target <code>/links/all/count</code>: <a href="/links/all/count?target={{ query.target|urlencode }}">/links/all/count?target={{ query.target }}</a></li> 20 + <li>See linking records to this target at <code>/links</code>: <a href="/links?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li> 21 + <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.target|urlencode }}">/links/all?target={{ query.target }}</a></li> 22 22 </ul> 23 23 24 24 <h3>DIDs, most recent first:</h3> 25 25 26 26 {% for did in linking_dids %} 27 27 <pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ did.0 }} 28 - -> see <a href="/links/all/count?target={{ did.0|urlencode }}">links to this DID</a> 28 + -> see <a href="/links/all?target={{ did.0|urlencode }}">links to this DID</a> 29 29 -> browse <a href="https://atproto-browser-plus-links.vercel.app/at/{{ did.0|urlencode }}">this DID record</a></pre> 30 30 {% endfor %} 31 31
+35
constellation/templates/explore-links.html.j2
··· 1 + {% extends "base.html.j2" %} 2 + {% import "try-it-macros.html.j2" as try_it %} 3 + 4 + {% block title %}Explore links{% endblock %} 5 + 6 + {% block content %} 7 + 8 + {% call try_it::explore_links(query.target) %} 9 + 10 + <h2> 11 + All links to <code>{{ query.target }}</code> 12 + {% if let Some(browseable_uri) = query.target|to_browseable %} 13 + <small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small> 14 + {% endif %} 15 + </h2> 16 + 17 + <h3>Links by collection and path:</h3> 18 + 19 + <pre style="display: block; margin: 1em 2em" class="code"> 20 + {%- for (collection, collection_links) in links -%} 21 + <strong>{{ collection }}</strong> 22 + {%- for (path, counts) in collection_links %} 23 + {{ path }}: <a href="/links?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.records }} links</a> from <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.distinct_dids }} distinct DIDs</a></li> 24 + {%- endfor %} 25 + 26 + {% else -%} 27 + <em>No links indexed for this target</em> 28 + {% endfor -%} 29 + </pre> 30 + <details> 31 + <summary>Raw JSON response</summary> 32 + <pre class="code">{{ self|tojson }}</pre> 33 + </details> 34 + 35 + {% endblock %}
+17 -1
constellation/templates/hello.html.j2
··· 80 80 {% call try_it::dids_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block", ".subject") %} 81 81 82 82 83 - <h3 class="route"><code>GET /links/all/count</code></h3> 83 + <h3 class="route"><code>GET /links/all</code></h3> 84 + 85 + <p>Show all sources with links to a target, including linking record counts and distinct linking DIDs</p> 86 + 87 + <h4>Query parameters:</h4> 88 + 89 + <ul> 90 + <li><code>target</code>: required, must url-encode. Example: <code>did:plc:oky5czdrnfjpqslsw2a5iclo</code></li> 91 + </ul> 92 + 93 + <p style="margin-bottom: 0"><strong>Try it:</strong></p> 94 + {% call try_it::explore_links("did:plc:oky5czdrnfjpqslsw2a5iclo") %} 95 + 96 + 97 + <h3 class="route deprecated"><code>[deprecated] GET /links/all/count</code></h3> 84 98 85 99 <p>The total counts of all links pointing at a given target, by collection and path.</p> 100 + 101 + <p>DEPRECATED: Use <code>GET /links/all</code> instead.</p> 86 102 87 103 <h4>Query parameters:</h4> 88 104
+3 -3
constellation/templates/links.html.j2
··· 17 17 <p><strong>{{ total }} links</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p> 18 18 19 19 <ul> 20 - <li>See distinct DIDs <code>/links/distinct-dids</code>: <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links/distinct-dids?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li> 21 - <li>See all links to this target <code>/links/all/count</code>: <a href="/links/all/count?target={{ query.target|urlencode }}">/links/all/count?target={{ query.target }}</a></li> 20 + <li>See distinct linking DIDs at <code>/links/distinct-dids</code>: <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links/distinct-dids?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li> 21 + <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.target|urlencode }}">/links/all?target={{ query.target }}</a></li> 22 22 </ul> 23 23 24 24 <h3>Links, most recent first:</h3> 25 25 26 26 {% for record in linking_records %} 27 - <pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ record.did().0 }} (<a href="/links/all/count?target={{ record.did().0|urlencode }}">DID links</a>) 27 + <pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ record.did().0 }} (<a href="/links/all?target={{ record.did().0|urlencode }}">DID links</a>) 28 28 <strong>Collection</strong>: {{ record.collection }} 29 29 <strong>RKey</strong>: {{ record.rkey }} 30 30 -> <a href="https://atproto-browser-plus-links.vercel.app/at/{{ record.did().0|urlencode }}/{{ record.collection }}/{{ record.rkey }}">browse record</a></pre>
+7
constellation/templates/try-it-macros.html.j2
··· 43 43 <pre class="code"><strong>GET</strong> /links/all/count?target=<input type="text" name="target" value="{{ target }}" placeholder="target" /> <button type="submit">get all target link counts</button></pre> 44 44 </form> 45 45 {% endmacro %} 46 + 47 + 48 + {% macro explore_links(target) %} 49 + <form method="get" action="/links/all"> 50 + <pre class="code"><strong>GET</strong> /links/all?target=<input type="text" name="target" value="{{ target }}" placeholder="target" /> <button type="submit">get all target link counts</button></pre> 51 + </form> 52 + {% endmacro %}