Shows how to get repo export and walk it in TypeScript
walktherepo.wisp.place
I think it's done?
+139
-27
+16
-3
src/App.svelte
+16
-3
src/App.svelte
···
6
6
let showRepoStats = $state(false);
7
7
let did = $state('');
8
8
let pdsUrl = $state('');
9
+
let slowPoke = $state(false);
9
10
10
-
const resolvedResult = (didResult: string, pdsUrlResult: string) => {
11
+
const resolvedResult = (didResult: string, pdsUrlResult: string, slowPokeResult: boolean) => {
11
12
did = didResult;
12
13
pdsUrl = pdsUrlResult;
14
+
slowPoke = slowPokeResult;
13
15
showRepoStats = true;
14
16
};
15
17
···
17
19
</script>
18
20
19
21
<main>
20
-
<h1>Repo Walk Example</h1>
22
+
{#if showRepoStats}
23
+
{#if slowPoke}
24
+
<h2>Walking the repo via api calls</h2>
25
+
{:else}
26
+
<h2>Walking the repo via repo export</h2>
27
+
{/if}
28
+
{:else}
29
+
<h1>Repo Walk Example</h1>
30
+
<br>
31
+
<p>Demo showing why you may rather export the users whole repo instead of walking it via api calls</p>
21
32
33
+
<sub>Also shows how many records you have and how many of each kind if you're into that kind of thing...</sub>
34
+
{/if}
22
35
<div class="card">
23
36
{#if showRepoStats}
24
-
<RepoStats did={did} pdsUrl={pdsUrl}/>
37
+
<RepoStats did={did} pdsUrl={pdsUrl} slowPokeMode={slowPoke}/>
25
38
{:else}
26
39
<SearchForm resolvedResult={resolvedResult}/>
27
40
{/if}
+119
-22
src/lib/RepoStats.svelte
+119
-22
src/lib/RepoStats.svelte
···
5
5
import { Client, simpleFetchHandler } from '@atcute/client';
6
6
import type {} from '@atcute/atproto';
7
7
8
-
const { did, pdsUrl } = $props();
8
+
const { did, pdsUrl, slowPokeMode } = $props();
9
9
10
10
interface CountedCollection {
11
11
collection: string;
12
12
count: number;
13
13
}
14
14
15
+
//Shared State
15
16
let loading = $state(true);
17
+
let error: string | null = $state(null);
18
+
//Downloaded stuff
16
19
let downloadedBytes = $state(0);
17
20
let downloadedMB = $derived((downloadedBytes / (1024 * 1024)).toFixed(2));
18
-
let error: string | null = $state(null);
21
+
//Ui counts for collections
19
22
let collections = $state(new Array<CountedCollection>());
20
23
let collectionsOrdered: Array<CountedCollection> = $derived([...collections].sort((a, b) => b.count - a.count));
21
24
let totalRecords = $state(0);
25
+
let currentCollection: string | null = $state(null);
26
+
//Timer stuff
22
27
let startTime = $state<number | null>(null);
23
28
let endTime = $state<number | null>(null);
24
29
let elapsedTime = $state('');
25
-
26
30
let interval = $state<number | null>(null);
31
+
//Just for the slow pokes
32
+
let webCalls = $state(0);
27
33
28
34
const calculateElapsedTime = () => {
29
35
if (!startTime) return '0.00';
···
32
38
};
33
39
34
40
const startTimer = () => {
41
+
endTime = null;
42
+
startTime = Date.now();
35
43
interval = setInterval(() => {
36
44
calculateElapsedTime();
37
45
}, 250);
38
46
};
39
47
48
+
const stopTimer = () => {
49
+
if (interval) {
50
+
clearInterval(interval);
51
+
}
52
+
endTime = Date.now();
53
+
calculateElapsedTime();
54
+
};
55
+
40
56
// Calls the getRepo endpoint to get a .car export to walk the repo
41
57
const getRepoStatsViaExport = async () => {
42
58
const rpc = new Client({ handler: simpleFetchHandler({ service: pdsUrl }) });
43
-
44
-
startTime = Date.now();
45
-
endTime = null;
46
59
startTimer();
47
60
try {
48
61
const result = await rpc.get('com.atproto.sync.getRepo', {
···
55
68
}
56
69
57
70
let stream = result.data;
58
-
59
-
60
71
const car = fromStream(stream);
61
72
62
73
try {
63
74
for await (const entry of car) {
64
-
const data = CBOR.decode(entry.bytes);
65
-
if (!data.$type) {
75
+
const record = CBOR.decode(entry.bytes);
76
+
if (!record.$type) {
77
+
//Is not a record
66
78
continue;
67
79
}
68
80
69
-
let checkForCollection = collections.find(c => c.collection === data.$type);
81
+
let checkForCollection = collections.find(c => c.collection === record.$type);
70
82
if (!checkForCollection) {
71
-
collections.push({ collection: data.$type, count: 1 });
83
+
collections.push({ collection: record.$type, count: 1 });
72
84
} else {
73
85
checkForCollection.count++;
74
86
}
···
76
88
totalRecords++;
77
89
}
78
90
} finally {
79
-
if (interval) {
80
-
clearInterval(interval);
81
-
interval = null;
82
-
calculateElapsedTime();
83
-
}
91
+
stopTimer();
84
92
await car.dispose();
85
93
}
86
94
87
-
endTime = Date.now();
88
95
loading = false;
89
96
} catch (err) {
90
-
endTime = Date.now();
97
+
stopTimer();
91
98
console.error('Error fetching repo stats:', err);
92
99
if (err instanceof Error) {
93
100
error = err.message;
···
98
105
}
99
106
};
100
107
108
+
//An ungodly amount of api calls showing the speed difference between getting an export and walking the repo
109
+
//via api calls
110
+
const getRepoStatsTheLongWay = async () => {
111
+
try {
112
+
const rpc = new Client({ handler: simpleFetchHandler({ service: pdsUrl }) });
113
+
startTimer();
114
+
let describeRepo = await rpc.get('com.atproto.repo.describeRepo', {
115
+
params: {
116
+
repo: did
117
+
}
118
+
});
119
+
if(!describeRepo.ok){
120
+
throw new Error(`HTTP error! status: ${describeRepo.status}`);
121
+
}
122
+
webCalls++;
123
+
for (const collection of describeRepo.data.collections) {
124
+
let totalRecordsInCollection = 0;
125
+
currentCollection = collection;
126
+
const firstCollectionList = await rpc.get('com.atproto.repo.listRecords', {
127
+
params: {
128
+
collection,
129
+
repo: did
130
+
}
131
+
});
132
+
webCalls++;
133
+
if(!firstCollectionList.ok){
134
+
console.error(`HTTP error! status: ${firstCollectionList.status}`);
135
+
continue;
136
+
}
137
+
totalRecords += firstCollectionList.data.records.length;
138
+
totalRecordsInCollection += firstCollectionList.data.records.length;
139
+
140
+
let cursor = firstCollectionList.data.cursor;
141
+
do {
142
+
const nextCollectionList = await rpc.get('com.atproto.repo.listRecords', {
143
+
params: {
144
+
collection,
145
+
repo: did,
146
+
cursor
147
+
}
148
+
});
149
+
webCalls++;
150
+
if(!nextCollectionList.ok){
151
+
console.error(`HTTP error! status: ${nextCollectionList.status}`);
152
+
continue;
153
+
}
154
+
totalRecordsInCollection += nextCollectionList.data.records.length;
155
+
cursor = nextCollectionList.data.cursor;
156
+
totalRecords += nextCollectionList.data.records.length;
157
+
158
+
} while (cursor !== undefined);
159
+
collections.push({ collection: collection, count:totalRecordsInCollection });
160
+
}
161
+
loading = false;
162
+
stopTimer();
163
+
}
164
+
catch (err) {
165
+
loading = false;
166
+
stopTimer();
167
+
console.error('Error fetching repo stats:', err);
168
+
if (err instanceof Error) {
169
+
error = err.message;
170
+
} else {
171
+
error = 'Unknown error: can check the console for more details';
172
+
}
173
+
}
174
+
};
175
+
101
176
onMount(() => {
102
-
getRepoStatsViaExport();
177
+
if (slowPokeMode) {
178
+
getRepoStatsTheLongWay();
179
+
} else {
180
+
getRepoStatsViaExport();
181
+
}
182
+
103
183
});
104
184
105
185
</script>
106
186
107
187
<div>
188
+
{#if slowPokeMode}
189
+
<img alt="A Shellder biting a Slowpoke's tail, as seen in the Pokémon anime "
190
+
src="https://upload.wikimedia.org/wikipedia/en/a/a2/Slowpoke_and_Shellder.jpg">
191
+
<br>
192
+
{/if}
193
+
108
194
{#if error}
109
195
<p style="color: red">{error}</p>
110
196
{/if}
111
-
{#if loading}
197
+
{#if loading && !slowPokeMode}
112
198
Loading... ({downloadedMB} MB downloaded, {elapsedTime}s)
199
+
{:else if loading && slowPokeMode}
200
+
Loading... ({webCalls.toLocaleString()} web calls made, {elapsedTime}s)
113
201
{:else}
114
-
<span>Repo size {downloadedMB} MB (fetched in {elapsedTime}s)</span>
202
+
{#if !slowPokeMode}
203
+
<span>Repo size {downloadedMB} MB (fetched in {elapsedTime}s)</span>
204
+
{:else}
205
+
<span>Web calls made: {webCalls.toLocaleString()} (fetched in {elapsedTime}s)</span>
206
+
{/if}
115
207
{/if}
208
+
{#if loading && currentCollection !== null}
209
+
<br>
210
+
<span>Currently walking collection: {currentCollection}</span>
211
+
{/if}
212
+
116
213
{#if collectionsOrdered.length > 0}
117
214
118
215
<br>
+4
-2
src/lib/SearchForm.svelte
+4
-2
src/lib/SearchForm.svelte
···
28
28
29
29
let handleToLookUp = $state('');
30
30
let error: string | null = $state(null);
31
+
let slowpoke = $state(false);
31
32
32
33
let { resolvedResult } = $props();
33
34
···
46
47
const didDoc = await didResolver.resolve(did);
47
48
const pdsUrl = getPdsEndpoint(didDoc);
48
49
49
-
resolvedResult(did, pdsUrl);
50
+
resolvedResult(did, pdsUrl, slowpoke);
50
51
}catch(e){
51
52
if (e instanceof Error) {
52
53
error = e.message;
···
63
64
<button>Lookup</button>
64
65
<br>
65
66
<label>
67
+
<input bind:checked={slowpoke} type="checkbox"/>
66
68
slowpoke (uses web calls to walk the repository to show you the speed difference)
67
-
<input type="checkbox"/>
69
+
68
70
</label>
69
71
{#if error}
70
72
<p style="color: red;">Error: {error}</p>