Shows how to get repo export and walk it in TypeScript
walktherepo.wisp.place
1<script lang="ts">
2 import { onMount } from 'svelte';
3 import { Client, simpleFetchHandler } from '@atcute/client';
4 import type {} from '@atcute/atproto';
5 import { fromStream } from '@atcute/repo';
6
7 const { did, handle, pdsUrl, slowPokeMode } = $props();
8
9 interface CountedCollection {
10 collection: string;
11 count: number;
12 }
13
14 //Shared State
15 let loading = $state(true);
16 let error: string | null = $state(null);
17 //Download info stuff
18 let downloadedBytes = $state(0);
19 let downloadedMB = $derived((downloadedBytes / (1024 * 1024)).toFixed(2));
20 //Ui counts for collections
21 let collections = $state(new Array<CountedCollection>());
22 let collectionsOrdered: Array<CountedCollection> = $derived([...collections].sort((a, b) => b.count - a.count));
23 let totalRecords = $state(0);
24 let currentCollection: string | null = $state(null);
25 //Timer stuff
26 let startTime = $state<number | null>(null);
27 let endTime = $state<number | null>(null);
28 let elapsedTime = $state('');
29 let interval = $state<number | null>(null);
30 //Just for the slow pokes
31 let webCalls = $state(0);
32
33 const calculateElapsedTime = () => {
34 if (!startTime) return '0.00';
35 const end = endTime ?? Date.now();
36 elapsedTime = ((end - startTime) / 1000).toFixed(2);
37 };
38
39 const startTimer = () => {
40 endTime = null;
41 startTime = Date.now();
42 interval = setInterval(() => {
43 calculateElapsedTime();
44 }, 250);
45 };
46
47 const stopTimer = () => {
48 if (interval) {
49 clearInterval(interval);
50 }
51 endTime = Date.now();
52 calculateElapsedTime();
53 };
54
55 // Calls the getRepo endpoint to get a .car export to walk the repo. allows you to stream and access records as they are downloaded
56 const getRepoStatsViaExport = async () => {
57 const rpc = new Client({ handler: simpleFetchHandler({ service: pdsUrl }) });
58 startTimer();
59 try {
60 const result = await rpc.get('com.atproto.sync.getRepo', {
61 params: { did: did },
62 as: 'stream',
63 });
64
65 if (!result.ok) {
66 throw new Error(`HTTP error! status: ${result.status}`);
67 }
68 const repo = fromStream(result.data);
69
70 try {
71 //This reads the repo as it is downloaded. which was very cool and I didn't know it would do that
72 for await (const entry of repo) {
73 // record here is the content of the atproto record
74 // console.log(entry.record);
75 let checkForCollection = collections.find(c => c.collection === entry.collection);
76 if (!checkForCollection) {
77 collections.push({ collection: entry.collection, count: 1 });
78 } else {
79 checkForCollection.count++;
80 }
81 downloadedBytes = entry.carEntry.entryEnd;
82 totalRecords++;
83 }
84 } finally {
85 stopTimer();
86 }
87 loading = false;
88 } catch (err) {
89 stopTimer();
90 console.log(err);
91 console.error('Error fetching repo stats:', err);
92 if (err instanceof Error) {
93 error = err.message;
94 } else {
95 error = 'Unknown error: can check the console for more details';
96 }
97 loading = false;
98 }
99 };
100
101 //An ungodly amount of api calls showing the speed difference between getting an export and walking the repo
102 //via api calls
103 const getRepoStatsTheLongWay = async () => {
104 try {
105 const rpc = new Client({ handler: simpleFetchHandler({ service: pdsUrl }) });
106 startTimer();
107 //We can make a call to get a list of collections in the repo
108 let describeRepo = await rpc.get('com.atproto.repo.describeRepo', {
109 params: {
110 repo: did
111 }
112 });
113 if(!describeRepo.ok){
114 throw new Error(`HTTP error! status: ${describeRepo.status}`);
115 }
116 webCalls++;
117 //We go through each collection and get a list of records in it
118 for (const collection of describeRepo.data.collections) {
119 let totalRecordsInCollection = 0;
120 currentCollection = collection;
121 //Do the first call to get a cursor
122 const firstCollectionList = await rpc.get('com.atproto.repo.listRecords', {
123 params: {
124 collection,
125 repo: did,
126 limit: 100,
127 }
128 });
129 webCalls++;
130 if(!firstCollectionList.ok){
131 console.error(`HTTP error! status: ${firstCollectionList.status}`);
132 continue;
133 }
134 totalRecords += firstCollectionList.data.records.length;
135 totalRecordsInCollection += firstCollectionList.data.records.length;
136
137 let cursor = firstCollectionList.data.cursor;
138 //Walk the collection till the cursor is undefined meaning no more records
139 do {
140 const nextCollectionList = await rpc.get('com.atproto.repo.listRecords', {
141 params: {
142 collection,
143 repo: did,
144 limit: 100,
145 cursor
146 }
147 });
148 webCalls++;
149 if(!nextCollectionList.ok){
150 console.error(`HTTP error! status: ${nextCollectionList.status}`);
151 continue;
152 }
153 totalRecordsInCollection += nextCollectionList.data.records.length;
154 cursor = nextCollectionList.data.cursor;
155 totalRecords += nextCollectionList.data.records.length;
156
157 } while (cursor !== undefined);
158 collections.push({ collection: collection, count:totalRecordsInCollection });
159 }
160 loading = false;
161 stopTimer();
162 }
163 catch (err) {
164 loading = false;
165 stopTimer();
166 console.error('Error fetching repo stats:', err);
167 if (err instanceof Error) {
168 error = err.message;
169 } else {
170 error = 'Unknown error: can check the console for more details';
171 }
172 }
173 };
174
175 onMount(() => {
176 if (slowPokeMode) {
177 getRepoStatsTheLongWay();
178 } else {
179 getRepoStatsViaExport();
180 }
181
182 });
183
184</script>
185
186<div class="flex flex-col items-center gap-4">
187 <div class="w-full flex justify-center">
188 {#if slowPokeMode}
189 <img
190 alt="A Shellder biting a Slowpoke's tail, as seen in the Pokémon anime"
191 src="/slowPoke.png"
192 class="max-w-sm rounded-lg shadow-lg"
193 >
194 {:else}
195 <img
196 alt="text in a speech bubble that says 'Dude, wheres my car'"
197 src="/dude.png"
198 class="max-w-sm rounded-lg shadow-lg"
199 >
200 {/if}
201 </div>
202
203 {#if error}
204 <div class="alert alert-error w-full">
205 <svg xmlns="http://www.w3.org/2000/svg" class="stroke-current shrink-0 h-6 w-6" fill="none" viewBox="0 0 24 24">
206 <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10 14l2-2m0 0l2-2m-2 2l-2-2m2 2l2 2m7-2a9 9 0 11-18 0 9 9 0 0118 0z" />
207 </svg>
208 <span>{error}</span>
209 </div>
210 {/if}
211
212 {#if loading && !slowPokeMode}
213 <div class="flex items-center gap-3">
214 <span class="loading loading-spinner loading-lg text-primary"></span>
215 <div class="text-lg">
216 <div class="font-semibold">Loading...</div>
217 <div class="text-sm opacity-70">
218 <span class="badge badge-info">{downloadedMB} MB</span> downloaded in
219 <span class="badge badge-ghost">{elapsedTime}s</span>
220 </div>
221 </div>
222 </div>
223 {:else if loading && slowPokeMode}
224 <div class="flex items-center gap-3">
225 <span class="loading loading-spinner loading-lg text-primary"></span>
226 <div class="text-lg">
227 <div class="font-semibold">Loading...</div>
228 <div class="text-sm opacity-70">
229 <span class="badge badge-info">{webCalls.toLocaleString()} web calls</span> made in
230 <span class="badge badge-ghost">{elapsedTime}s</span>
231 </div>
232 </div>
233 </div>
234 {:else}
235 <div class="stats shadow bg-base-300">
236 <div class="stat">
237 <div class="stat-title">{slowPokeMode ? 'Web Calls Made' : 'Repo Size'}</div>
238 <div class="stat-value text-primary">
239 {#if !slowPokeMode}
240 {downloadedMB} MB
241 {:else}
242 {webCalls.toLocaleString()}
243 {/if}
244 </div>
245 <div class="stat-desc">Fetched in {elapsedTime}s</div>
246 </div>
247 </div>
248 {/if}
249
250 {#if loading && currentCollection !== null}
251 <div class="alert alert-info">
252 <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" class="stroke-current shrink-0 w-6 h-6">
253 <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"></path>
254 </svg>
255 <span>Currently walking collection: <strong>{currentCollection}</strong></span>
256 </div>
257 {/if}
258
259 {#if collectionsOrdered.length > 0}
260 <div class="stats stats-vertical lg:stats-horizontal shadow bg-base-300 w-full">
261 <div class="stat">
262 <div class="stat-title">Total Records</div>
263 <div class="stat-value text-secondary">{totalRecords.toLocaleString()}</div>
264 </div>
265 <div class="stat">
266 <div class="stat-title">Different Collections</div>
267 <div class="stat-value text-accent">{collectionsOrdered.length}</div>
268 </div>
269 </div>
270
271 <div class="card bg-base-300 shadow-xl w-full">
272 <div class="card-body">
273 <h3 class="card-title">{handle}'s Collections Breakdown</h3>
274 <ol class="list-decimal list-inside space-y-2">
275 {#each collectionsOrdered as collection (collection.collection)}
276 <li class="text-sm">
277 <a class="link font-mono text-primary" href="https://pdsls.dev/at://{did}/{collection.collection}" target="_blank">{collection.collection}</a>
278 <span class="badge badge-sm ml-2">{collection.count.toLocaleString()} records</span>
279 </li>
280 {/each}
281 </ol>
282 </div>
283 </div>
284 {/if}
285</div>