open source is social v-it.org
Merge branch 'hopper-zxptvgvd-cross-pds-and-scan'
+196
-26
+2
src/cli.js
+2
src/cli.js
···
10
10
import registerInit from './cmd/init.js';
11
11
import registerLogin from './cmd/login.js';
12
12
import registerFirehose from './cmd/firehose.js';
13
+
import registerScan from './cmd/scan.js';
13
14
import registerRemix from './cmd/remix.js';
14
15
import registerShip from './cmd/ship.js';
15
16
import registerSkim from './cmd/skim.js';
···
33
34
registerInit(program);
34
35
registerLogin(program);
35
36
registerFirehose(program);
37
+
registerScan(program);
36
38
registerShip(program);
37
39
registerSkim(program);
38
40
registerRemix(program);
+10
-1
src/cmd/firehose.js
+10
-1
src/cmd/firehose.js
···
113
113
.description('Listen to Jetstream for cap events')
114
114
.option('-v, --verbose', 'Show full JSON for each event')
115
115
.option('--did <did>', 'Filter by DID (reads saved DID from config if not provided)')
116
+
.option('--global', 'Show cap events from all DIDs across the network')
116
117
.option('--collection <nsid>', 'Collection NSID to filter', CAP_COLLECTION)
117
118
.action(async (opts) => {
118
119
try {
119
-
if (!opts.did) {
120
+
if (opts.global && opts.did) {
121
+
console.error('error: --global and --did are mutually exclusive');
122
+
process.exitCode = 1;
123
+
return;
124
+
}
125
+
126
+
if (opts.global) {
127
+
opts.did = undefined;
128
+
} else if (!opts.did) {
120
129
const config = loadConfig();
121
130
if (config.did) {
122
131
opts.did = config.did;
+5
-6
src/cmd/remix.js
+5
-6
src/cmd/remix.js
···
8
8
import { requireAgent } from '../lib/agent.js';
9
9
import { resolveRef, REF_PATTERN } from '../lib/cap-ref.js';
10
10
import { brand, name } from '../lib/brand.js';
11
+
import { resolvePds, listRecordsFromPds } from '../lib/pds.js';
11
12
12
13
export default function register(program) {
13
14
program
···
73
74
let match = null;
74
75
for (const repoDid of dids) {
75
76
try {
76
-
const res = await agent.com.atproto.repo.listRecords({
77
-
repo: repoDid,
78
-
collection: CAP_COLLECTION,
79
-
limit: 50,
80
-
});
81
-
for (const rec of res.data.records) {
77
+
const pds = await resolvePds(repoDid);
78
+
if (verbose) console.log(`[verbose] ${repoDid}: resolved PDS ${pds}`);
79
+
const res = await listRecordsFromPds(pds, repoDid, CAP_COLLECTION, 50);
80
+
for (const rec of res.records) {
82
81
if (rec.value.beacon !== beacon) continue;
83
82
const recRef = resolveRef(rec.value, rec.cid);
84
83
if (recRef === ref) {
+117
src/cmd/scan.js
+117
src/cmd/scan.js
···
1
+
// SPDX-License-Identifier: AGPL-3.0-only
2
+
// Copyright (c) 2026 sol pbc
3
+
4
+
import { CAP_COLLECTION } from '../lib/constants.js';
5
+
import { resolveRef } from '../lib/cap-ref.js';
6
+
import { resolveHandleFromDid } from '../lib/pds.js';
7
+
import { brand } from '../lib/brand.js';
8
+
9
+
const JETSTREAM_URL = 'wss://jetstream2.us-east.bsky.network/subscribe';
10
+
11
+
export default function register(program) {
12
+
program
13
+
.command('scan')
14
+
.description('Discover cap publishers across the network via Jetstream replay')
15
+
.option('--days <n>', 'Number of days to replay', '7')
16
+
.option('--beacon <beacon>', 'Filter by beacon')
17
+
.option('-v, --verbose', 'Show each event as it arrives')
18
+
.action(async (opts) => {
19
+
try {
20
+
const days = parseInt(opts.days, 10);
21
+
if (isNaN(days) || days < 1) {
22
+
console.error('error: --days must be a positive integer');
23
+
process.exitCode = 1;
24
+
return;
25
+
}
26
+
27
+
const cursor = (Date.now() - days * 86400000) * 1000;
28
+
const timeout = Math.max(120000, Math.min(600000, days * 60000));
29
+
30
+
const url = new URL(JETSTREAM_URL);
31
+
url.searchParams.set('wantedCollections', CAP_COLLECTION);
32
+
url.searchParams.set('cursor', String(cursor));
33
+
34
+
console.log(`${brand} scan`);
35
+
console.log(` Replaying ${days} day${days === 1 ? '' : 's'} of cap events...`);
36
+
if (opts.beacon) console.log(` Beacon filter: ${opts.beacon}`);
37
+
console.log(` Timeout: ${Math.round(timeout / 1000)}s`);
38
+
console.log('');
39
+
40
+
const publishers = new Map();
41
+
42
+
await new Promise((resolve, reject) => {
43
+
const ws = new WebSocket(url.toString());
44
+
const timer = setTimeout(() => {
45
+
ws.close();
46
+
resolve();
47
+
}, timeout);
48
+
49
+
ws.onmessage = (event) => {
50
+
let msg;
51
+
try { msg = JSON.parse(event.data); } catch { return; }
52
+
53
+
if (msg.kind !== 'commit' || msg.commit?.operation !== 'create') return;
54
+
55
+
const record = msg.commit?.record;
56
+
if (!record) return;
57
+
58
+
if (opts.beacon && record.beacon !== opts.beacon) return;
59
+
60
+
const did = msg.did;
61
+
const ref = msg.commit?.cid ? resolveRef(record, msg.commit.cid) : null;
62
+
63
+
if (opts.verbose) {
64
+
const didShort = did.slice(-12);
65
+
const title = record.title || '';
66
+
const refPart = ref ? ` (${ref})` : '';
67
+
console.log(` ${didShort}: ${title}${refPart} [${record.beacon || 'no beacon'}]`);
68
+
}
69
+
70
+
if (!publishers.has(did)) {
71
+
publishers.set(did, { count: 0, beacons: new Set(), lastActive: '' });
72
+
}
73
+
const entry = publishers.get(did);
74
+
entry.count++;
75
+
if (record.beacon) entry.beacons.add(record.beacon);
76
+
if (record.createdAt && record.createdAt > entry.lastActive) {
77
+
entry.lastActive = record.createdAt;
78
+
}
79
+
};
80
+
81
+
ws.onerror = (err) => {
82
+
clearTimeout(timer);
83
+
reject(new Error(`WebSocket error: ${err?.message ?? 'unknown'}`));
84
+
};
85
+
86
+
ws.onclose = () => {
87
+
clearTimeout(timer);
88
+
resolve();
89
+
};
90
+
});
91
+
92
+
if (publishers.size === 0) {
93
+
console.log('no cap publishers found in this time window.');
94
+
return;
95
+
}
96
+
97
+
const entries = [];
98
+
for (const [did, stats] of publishers) {
99
+
const handle = await resolveHandleFromDid(did);
100
+
entries.push({ handle, did, ...stats, beacons: [...stats.beacons] });
101
+
}
102
+
103
+
entries.sort((a, b) => b.count - a.count);
104
+
105
+
console.log(`found ${entries.length} publisher${entries.length === 1 ? '' : 's'}:\n`);
106
+
for (const e of entries) {
107
+
const beaconStr = e.beacons.length > 0 ? e.beacons.join(', ') : '(none)';
108
+
const lastActive = e.lastActive ? e.lastActive.split('T')[0] : 'unknown';
109
+
console.log(` @${e.handle}`);
110
+
console.log(` caps: ${e.count} beacons: ${beaconStr} last active: ${lastActive}`);
111
+
}
112
+
} catch (err) {
113
+
console.error(err instanceof Error ? err.message : String(err));
114
+
process.exitCode = 1;
115
+
}
116
+
});
117
+
}
+5
-6
src/cmd/ship.js
+5
-6
src/cmd/ship.js
···
10
10
import { appendLog, readProjectConfig, readLog, readFollowing } from '../lib/vit-dir.js';
11
11
import { REF_PATTERN, resolveRef } from '../lib/cap-ref.js';
12
12
import { name } from '../lib/brand.js';
13
+
import { resolvePds, listRecordsFromPds } from '../lib/pds.js';
13
14
14
15
export default function register(program) {
15
16
program
···
104
105
let match = null;
105
106
for (const repoDid of dids) {
106
107
try {
107
-
const res = await agent.com.atproto.repo.listRecords({
108
-
repo: repoDid,
109
-
collection: CAP_COLLECTION,
110
-
limit: 50,
111
-
});
112
-
for (const rec of res.data.records) {
108
+
const pds = await resolvePds(repoDid);
109
+
if (verbose) console.log(`[verbose] ${repoDid}: resolved PDS ${pds}`);
110
+
const res = await listRecordsFromPds(pds, repoDid, CAP_COLLECTION, 50);
111
+
for (const rec of res.records) {
113
112
const recRef = resolveRef(rec.value, rec.cid);
114
113
if (recRef === opts.recap) {
115
114
if (!match || (rec.value.createdAt || '') > (match.value.createdAt || '')) {
+6
-7
src/cmd/skim.js
+6
-7
src/cmd/skim.js
···
8
8
import { requireAgent } from '../lib/agent.js';
9
9
import { resolveRef } from '../lib/cap-ref.js';
10
10
import { name } from '../lib/brand.js';
11
+
import { resolvePds, listRecordsFromPds } from '../lib/pds.js';
11
12
12
13
export default function register(program) {
13
14
program
···
76
77
const allCaps = [];
77
78
for (const repoDid of dids) {
78
79
try {
79
-
const res = await agent.com.atproto.repo.listRecords({
80
-
repo: repoDid,
81
-
collection: CAP_COLLECTION,
82
-
limit: 50,
83
-
});
84
-
const caps = res.data.records.filter(r => r.value.beacon === beacon);
85
-
if (verbose) console.log(`[verbose] ${repoDid}: ${res.data.records.length} caps, ${caps.length} matching beacon`);
80
+
const pds = await resolvePds(repoDid);
81
+
if (verbose) console.log(`[verbose] ${repoDid}: resolved PDS ${pds}`);
82
+
const res = await listRecordsFromPds(pds, repoDid, CAP_COLLECTION, 50);
83
+
const caps = res.records.filter(r => r.value.beacon === beacon);
84
+
if (verbose) console.log(`[verbose] ${repoDid}: ${res.records.length} caps, ${caps.length} matching beacon`);
86
85
for (const cap of caps) cap._handle = handleMap.get(repoDid) || repoDid;
87
86
allCaps.push(...caps);
88
87
} catch (err) {
+5
-6
src/cmd/vet.js
+5
-6
src/cmd/vet.js
···
8
8
import { requireNotAgent } from '../lib/agent.js';
9
9
import { resolveRef, REF_PATTERN } from '../lib/cap-ref.js';
10
10
import { mark, brand, name } from '../lib/brand.js';
11
+
import { resolvePds, listRecordsFromPds } from '../lib/pds.js';
11
12
12
13
export default function register(program) {
13
14
program
···
71
72
let match = null;
72
73
for (const repoDid of dids) {
73
74
try {
74
-
const res = await agent.com.atproto.repo.listRecords({
75
-
repo: repoDid,
76
-
collection: CAP_COLLECTION,
77
-
limit: 50,
78
-
});
79
-
for (const rec of res.data.records) {
75
+
const pds = await resolvePds(repoDid);
76
+
if (verbose) console.log(`[verbose] ${repoDid}: resolved PDS ${pds}`);
77
+
const res = await listRecordsFromPds(pds, repoDid, CAP_COLLECTION, 50);
78
+
for (const rec of res.records) {
80
79
if (rec.value.beacon !== beacon) continue;
81
80
const recRef = resolveRef(rec.value, rec.cid);
82
81
if (recRef === ref) {
+34
src/lib/pds.js
+34
src/lib/pds.js
···
1
+
// SPDX-License-Identifier: AGPL-3.0-only
2
+
// Copyright (c) 2026 sol pbc
3
+
4
+
const PLC_DIRECTORY = 'https://plc.directory';
5
+
const pdsCache = new Map();
6
+
7
+
export async function resolvePds(did) {
8
+
if (pdsCache.has(did)) return pdsCache.get(did);
9
+
const res = await fetch(`${PLC_DIRECTORY}/${did}`);
10
+
if (!res.ok) throw new Error(`failed to resolve PDS for ${did}: ${res.status} ${res.statusText}`);
11
+
const doc = await res.json();
12
+
const pds = doc.service?.find(s => s.type === 'AtprotoPersonalDataServer');
13
+
if (!pds?.serviceEndpoint) throw new Error(`no PDS found in DID document for ${did}`);
14
+
pdsCache.set(did, pds.serviceEndpoint);
15
+
return pds.serviceEndpoint;
16
+
}
17
+
18
+
export async function listRecordsFromPds(pdsUrl, repo, collection, limit) {
19
+
const url = new URL('/xrpc/com.atproto.repo.listRecords', pdsUrl);
20
+
url.searchParams.set('repo', repo);
21
+
url.searchParams.set('collection', collection);
22
+
if (limit) url.searchParams.set('limit', String(limit));
23
+
const res = await fetch(url);
24
+
if (!res.ok) throw new Error(`listRecords failed for ${repo}: ${res.status} ${res.statusText}`);
25
+
return res.json();
26
+
}
27
+
28
+
export async function resolveHandleFromDid(did) {
29
+
const res = await fetch(`${PLC_DIRECTORY}/${did}`);
30
+
if (!res.ok) return did;
31
+
const doc = await res.json();
32
+
const aka = doc.alsoKnownAs?.find(a => a.startsWith('at://'));
33
+
return aka ? aka.replace('at://', '') : did;
34
+
}
+12
test/ship.test.js
+12
test/ship.test.js
···
3
3
4
4
import { describe, test, expect } from 'bun:test';
5
5
import { run } from './helpers.js';
6
+
import { mkdirSync, rmSync } from 'node:fs';
7
+
import { tmpdir } from 'node:os';
8
+
import { join } from 'node:path';
6
9
7
10
const agentEnv = { CLAUDECODE: '1' };
8
11
···
66
69
expect(r.exitCode).not.toBe(0);
67
70
expect(r.stderr).not.toMatch(/three lowercase words/i);
68
71
expect(r.stderr).not.toMatch(/body is required/i);
72
+
});
73
+
74
+
test('errors when no beacon set', () => {
75
+
const tmp = join(tmpdir(), '.test-ship-beacon-' + Math.random().toString(36).slice(2));
76
+
mkdirSync(tmp, { recursive: true });
77
+
const r = run('ship --title "Hi" --description "desc" --ref "one-two-three" --did "did:plc:abc"', tmp, agentEnv, 'body text');
78
+
expect(r.exitCode).not.toBe(0);
79
+
expect(r.stderr).toContain('no beacon set');
80
+
rmSync(tmp, { recursive: true, force: true });
69
81
});
70
82
});