very fast at protocol indexer with flexible filtering, xrpc queries, cursor-backed event stream, and more, built on fjall
rust
fjall
at-protocol
atproto
indexer
[tests] add run_all for running tests in parallel
+331
-248
+10
-9
AGENTS.md
+10
-9
AGENTS.md
···
109
109
## Safe commands
110
110
111
111
### Testing
112
-
- `nu tests/repo_sync_integrity.nu` - Runs the full integration test suite using Nushell. This builds the binary, starts a temporary instance, performs a backfill against a real PDS, and verifies record integrity.
112
+
- `nu tests/run_all.nu` - Runs all tests in parallel with automatically assigned free ports. Pass `--skip-creds` to skip tests requiring `.env` credentials, or `--only=[<name>...]` to run a subset.
113
+
- `nu tests/repo_sync_integrity.nu` - Performs a backfill against a real PDS, and verifies record integrity compared to hydrant.
113
114
- `nu tests/verify_crawler.nu` - Verifies full-network crawler functionality using a mock relay.
114
-
- `nu tests/throttling_test.nu` - Verifies crawler throttling logic when pending queue is full.
115
-
- `nu tests/stream_test.nu` - Tests WebSocket streaming functionality. Verifies both live event streaming during backfill and historical replay with cursor.
116
-
- `nu tests/authenticated_stream_test.nu` - Tests authenticated event streaming. Verifies that create, update, and delete actions on a real account are correctly streamed by Hydrant in the correct order. Requires `TEST_REPO` and `TEST_PASSWORD` in `.env`.
115
+
- `nu tests/throttling.nu` - Verifies crawler throttling logic when pending queue is full.
116
+
- `nu tests/stream.nu` - Tests WebSocket streaming functionality. Verifies both live event streaming during backfill and historical replay with cursor.
117
+
- `nu tests/authenticated_stream.nu` - Tests authenticated event streaming. Verifies that create, update, and delete actions on a real account are correctly streamed by Hydrant in the correct order. Requires `TEST_REPO` and `TEST_PASSWORD` in `.env`.
117
118
- `nu tests/debug_endpoints.nu` - Tests debug/introspection endpoints (`/debug/iter`, `/debug/get`) and verifies DB content and serialization.
118
-
- `nu tests/api_test.nu` - Tests management API endpoints (filter, repos, ingestion, sources).
119
-
- `nu tests/repos_api_test.nu` - Tests the `/repos` API endpoints including pagination and single-repo lookup.
120
-
- `nu tests/signal_filter_test.nu` - Verifies signal-based filtered indexing.
121
-
- `nu tests/collection_index_test.nu` - Tests collection-indexed crawling via `listReposByCollection`.
122
-
- `nu tests/backlinks_test.nu` - Tests backlinks indexing and XRPC query endpoints (requires `backlinks` feature).
119
+
- `nu tests/api.nu` - Tests management API endpoints (filter, repos, ingestion, sources).
120
+
- `nu tests/repos_api.nu` - Tests the `/repos` API endpoints including pagination and single-repo lookup.
121
+
- `nu tests/signal_filter.nu` - Verifies signal-based filtered indexing.
122
+
- `nu tests/by_collection.nu` - Tests by-collection crawling via `listReposByCollection`.
123
+
- `nu tests/backlinks.nu` - Tests backlinks indexing and XRPC query endpoints (requires `backlinks` feature).
123
124
- `nu tests/ephemeral_gc.nu` - Tests ephemeral mode TTL expiration and event watermark cleanup.
124
125
125
126
## Rust code style
+22
-58
tests/api_test.nu
tests/api.nu
+22
-58
tests/api_test.nu
tests/api.nu
···
1
1
#!/usr/bin/env nu
2
2
use common.nu *
3
3
4
-
# print a failure message, kill any running hydrant instances, and exit.
5
-
def fail [msg: string, ...pids: int] {
6
-
print $" FAILED: ($msg)"
7
-
for pid in $pids {
8
-
try { kill $pid }
9
-
}
10
-
exit 1
11
-
}
12
-
13
4
def test-crawler-sources [url: string, pid: int] {
14
5
print "=== test: crawler sources ==="
15
6
···
23
14
24
15
# add a relay source
25
16
print " POST /crawler/sources (relay)..."
26
-
let resp_add = (http post -f -e -t application/json $"($url)/crawler/sources" {
17
+
http post -f -e -t application/json $"($url)/crawler/sources" {
27
18
url: "https://bsky.network",
28
19
mode: "relay"
29
-
})
30
-
if $resp_add.status != 201 {
31
-
fail $"expected 201, got ($resp_add.status)" $pid
32
-
}
20
+
} | assert-status 201 "POST /crawler/sources" $pid
33
21
print " ok: 201 Created"
34
22
35
23
# verify the source appears with correct fields
···
49
37
50
38
# posting the same URL with a different mode replaces the existing entry
51
39
print " POST /crawler/sources (should override)..."
52
-
let resp_replace = (http post -f -e -t application/json $"($url)/crawler/sources" {
40
+
http post -f -e -t application/json $"($url)/crawler/sources" {
53
41
url: "https://bsky.network",
54
42
mode: "by_collection"
55
-
})
56
-
if $resp_replace.status != 201 {
57
-
fail $"expected 201, got ($resp_replace.status)" $pid
58
-
}
43
+
} | assert-status 201 "POST /crawler/sources override" $pid
59
44
let after_replace = (http get $"($url)/crawler/sources")
60
45
if ($after_replace | length) != 1 {
61
46
fail $"expected 1 source after override, got ($after_replace | length)" $pid
···
67
52
68
53
# remove the source
69
54
print " DELETE /crawler/sources..."
70
-
let resp_del = (http delete -f -e -t application/json $"($url)/crawler/sources" --data {
55
+
http delete -f -e -t application/json $"($url)/crawler/sources" --data {
71
56
url: "https://bsky.network"
72
-
})
73
-
if $resp_del.status != 200 {
74
-
fail $"expected 200, got ($resp_del.status)" $pid
75
-
}
57
+
} | assert-status 200 "DELETE /crawler/sources" $pid
76
58
let after_del = (http get $"($url)/crawler/sources")
77
59
if ($after_del | length) != 0 {
78
60
fail "expected empty list after delete" $pid
···
81
63
82
64
# deleting a non-existent source returns 404
83
65
print " DELETE /crawler/sources (should be 404)..."
84
-
let resp_del_missing = (http delete -f -e -t application/json $"($url)/crawler/sources" --data {
66
+
http delete -f -e -t application/json $"($url)/crawler/sources" --data {
85
67
url: "https://bsky.network"
86
-
})
87
-
if $resp_del_missing.status != 404 {
88
-
fail $"expected 404, got ($resp_del_missing.status)" $pid
89
-
}
68
+
} | assert-status 404 "DELETE /crawler/sources missing" $pid
90
69
print " ok: 404 for non-existent source"
91
70
92
71
print "crawler source tests passed!"
···
175
154
176
155
# the task can be stopped at runtime
177
156
print " deleting config source at runtime..."
178
-
let resp = (http delete -f -e -t application/json $"($url)/crawler/sources" --data {
157
+
http delete -f -e -t application/json $"($url)/crawler/sources" --data {
179
158
url: $crawler_url
180
-
})
181
-
if $resp.status != 200 {
182
-
fail $"expected 200, got ($resp.status)" $instance.pid
183
-
}
159
+
} | assert-status 200 "DELETE /crawler/sources runtime" $instance.pid
184
160
let after_del = (http get $"($url)/crawler/sources")
185
161
if ($after_del | length) != 0 {
186
162
fail "expected source to be gone after runtime delete" $instance.pid
···
225
201
226
202
# add a relay source
227
203
print " POST /firehose/sources..."
228
-
let resp_add = (http post -f -e -t application/json $"($url)/firehose/sources" {
204
+
http post -f -e -t application/json $"($url)/firehose/sources" {
229
205
url: "wss://test.bsky.network"
230
-
})
231
-
if $resp_add.status != 201 {
232
-
fail $"expected 201, got ($resp_add.status)" $pid
233
-
}
206
+
} | assert-status 201 "POST /firehose/sources" $pid
234
207
print " ok: 201 Created"
235
208
236
209
# verify it appears
···
247
220
248
221
# posting the same URL replaces the existing entry
249
222
print " POST /firehose/sources (should override)..."
250
-
let resp_replace = (http post -f -e -t application/json $"($url)/firehose/sources" {
223
+
http post -f -e -t application/json $"($url)/firehose/sources" {
251
224
url: "wss://test.bsky.network"
252
-
})
253
-
if $resp_replace.status != 201 {
254
-
fail $"expected 201, got ($resp_replace.status)" $pid
255
-
}
225
+
} | assert-status 201 "POST /firehose/sources override" $pid
256
226
let after_replace = (http get $"($url)/firehose/sources")
257
227
if ($after_replace | length) != 1 {
258
228
fail $"expected 1 source after override, got ($after_replace | length)" $pid
···
261
231
262
232
# remove the source
263
233
print " DELETE /firehose/sources..."
264
-
let resp_del = (http delete -f -e -t application/json $"($url)/firehose/sources" --data {
234
+
http delete -f -e -t application/json $"($url)/firehose/sources" --data {
265
235
url: "wss://test.bsky.network"
266
-
})
267
-
if $resp_del.status != 200 {
268
-
fail $"expected 200, got ($resp_del.status)" $pid
269
-
}
236
+
} | assert-status 200 "DELETE /firehose/sources" $pid
270
237
let after_del = (http get $"($url)/firehose/sources")
271
238
if ($after_del | length) != 0 {
272
239
fail "expected empty list after delete" $pid
···
275
242
276
243
# deleting a non-existent source returns 404
277
244
print " DELETE /firehose/sources (should be 404)..."
278
-
let resp_del_missing = (http delete -f -e -t application/json $"($url)/firehose/sources" --data {
245
+
http delete -f -e -t application/json $"($url)/firehose/sources" --data {
279
246
url: "wss://test.bsky.network"
280
-
})
281
-
if $resp_del_missing.status != 404 {
282
-
fail $"expected 404, got ($resp_del_missing.status)" $pid
283
-
}
247
+
} | assert-status 404 "DELETE /firehose/sources missing" $pid
284
248
print " ok: 404 for non-existent source"
285
249
286
250
print "firehose source tests passed!"
287
251
}
288
252
289
253
def main [] {
290
-
let port = 3007
254
+
let port = resolve-test-port 3007
291
255
let url = $"http://localhost:($port)"
292
256
293
257
let binary = build-hydrant
294
258
295
-
let db = (mktemp -d -t hydrant_api_test.XXXXXX)
259
+
let db = (mktemp -d -t hydrant_api.XXXXXX)
296
260
print $"db: ($db)"
297
261
298
262
let instance = (with-env { HYDRANT_CRAWLER_URLS: "", HYDRANT_RELAY_HOSTS: "" } {
···
308
272
kill $instance.pid
309
273
sleep 2sec
310
274
311
-
let db_persist = (mktemp -d -t hydrant_api_test.XXXXXX)
275
+
let db_persist = (mktemp -d -t hydrant_api.XXXXXX)
312
276
print $"db: ($db_persist)"
313
277
test-source-persistence $binary $db_persist $port
314
278
315
279
sleep 1sec
316
280
317
-
let db_config = (mktemp -d -t hydrant_api_test.XXXXXX)
281
+
let db_config = (mktemp -d -t hydrant_api.XXXXXX)
318
282
print $"db: ($db_config)"
319
283
test-config-source-not-persisted $binary $db_config $port
320
284
+8
-6
tests/authenticated_stream_test.nu
tests/authenticated_stream.nu
+8
-6
tests/authenticated_stream_test.nu
tests/authenticated_stream.nu
···
36
36
} catch {
37
37
print "warning: failed to add repo (might already be tracked), continuing..."
38
38
}
39
-
40
-
sleep 5sec
39
+
wait-for-backfill $url
41
40
42
41
# 5. perform actions
43
42
let collection = "app.bsky.feed.post"
···
91
90
92
91
# 6. verify
93
92
sleep 3sec
93
+
94
94
print "stopping listener..."
95
95
try { kill -9 $stream_pid }
96
96
···
173
173
let password = ($env_vars | get --optional TEST_PASSWORD)
174
174
175
175
if ($did | is-empty) or ($password | is-empty) {
176
-
print "error: TEST_REPO and TEST_PASSWORD must be set in .env"
177
-
exit 1
176
+
print "SKIP: TEST_REPO and TEST_PASSWORD not set in .env"
177
+
exit 0
178
178
}
179
179
180
180
let pds_url = resolve-pds $did
···
182
182
# ensure build
183
183
build-hydrant | ignore
184
184
185
+
let port = resolve-test-port 3005
186
+
185
187
print "=== running single-relay test ==="
186
188
let relay1 = "wss://relay.fire.hose.cam"
187
-
let success1 = run-auth-test $did $password $pds_url $relay1 3005
189
+
let success1 = run-auth-test $did $password $pds_url $relay1 $port
188
190
189
191
print ""
190
192
print "=== running multi-relay test ==="
191
193
let relay_multi = "wss://relay.fire.hose.cam,wss://relay3.fr.hose.cam,wss://relay1.us-west.bsky.network,wss://relay1.us-east.bsky.network"
192
-
let success2 = run-auth-test $did $password $pds_url $relay_multi 3015
194
+
let success2 = run-auth-test $did $password $pds_url $relay_multi $port
193
195
194
196
if $success1 and $success2 {
195
197
print ""
+1
-1
tests/backlinks_test.nu
tests/backlinks.nu
+1
-1
tests/backlinks_test.nu
tests/backlinks.nu
+1
-1
tests/collection_index_test.nu
tests/by_collection.nu
+1
-1
tests/collection_index_test.nu
tests/by_collection.nu
+49
-1
tests/common.nu
+49
-1
tests/common.nu
···
1
+
# print a failure message, kill any running hydrant pids, and exit with code 1.
2
+
export def fail [msg: string, ...pids: int] {
3
+
print $" FAILED: ($msg)"
4
+
for pid in $pids {
5
+
try { kill $pid }
6
+
}
7
+
exit 1
8
+
}
9
+
10
+
# pipe a full http response (obtained via -f -e flags) through this to assert an expected
11
+
# status code. on mismatch, prints the status, response body, kills any supplied pids, and exits.
12
+
# on success, returns the response record so callers can inspect fields further.
13
+
export def assert-status [expected: int, label: string, ...pids: int] {
14
+
let resp = $in
15
+
if $resp.status != $expected {
16
+
print $" ($label): expected status ($expected), got ($resp.status)"
17
+
try { print $" body: ($resp.body)" }
18
+
for pid in $pids { try { kill $pid } }
19
+
exit 1
20
+
}
21
+
$resp
22
+
}
23
+
24
+
# resolve the api port this test instance should use.
25
+
export def resolve-test-port [default: int] {
26
+
$env | get --optional HYDRANT_API_PORT | default ($default | into string) | into int
27
+
}
28
+
29
+
# resolve the debug port this test instance should use.
30
+
export def resolve-test-debug-port [default: int] {
31
+
$env | get --optional HYDRANT_DEBUG_PORT | default ($default | into string) | into int
32
+
}
33
+
34
+
# resolve the mock relay port for tests that need one.
35
+
export def resolve-test-mock-port [default: int] {
36
+
$env | get --optional HYDRANT_TEST_MOCK_PORT | default ($default | into string) | into int
37
+
}
38
+
39
+
export def resolve-binary [default: string] {
40
+
$env | get --optional HYDRANT_BINARY | default $default
41
+
}
42
+
1
43
export def load-env-file [] {
2
44
if (".env" | path exists) {
3
45
let content = (open .env)
···
51
93
52
94
# build the hydrant binary
53
95
export def build-hydrant [] {
96
+
if ($env | get --optional HYDRANT_BINARY | is-not-empty) {
97
+
return $env.HYDRANT_BINARY
98
+
}
54
99
print "building hydrant..."
55
100
cargo build
56
101
"target/debug/hydrant"
···
58
103
59
104
# build the hydrant binary with extra cargo features (space-separated string)
60
105
export def build-hydrant-features [features: string] {
106
+
if ($env | get --optional HYDRANT_BINARY | is-not-empty) {
107
+
return $env.HYDRANT_BINARY
108
+
}
61
109
print $"building hydrant with features: ($features)..."
62
110
cargo build --features $features
63
111
"target/debug/hydrant"
···
74
122
HYDRANT_FULL_NETWORK: "false",
75
123
HYDRANT_API_PORT: ($port | into string),
76
124
HYDRANT_ENABLE_DEBUG: "true",
77
-
HYDRANT_DEBUG_PORT: ($port + 1 | into string),
125
+
HYDRANT_DEBUG_PORT: (resolve-test-debug-port ($port + 1) | into string),
78
126
HYDRANT_PLC_URL: "https://plc.gaze.systems",
79
127
RUST_LOG: "debug,hyper=error,tokio=error,h2=error,tower=error,rustls=error"
80
128
} | merge $hydrant_vars
+8
-8
tests/debug_endpoints.nu
+8
-8
tests/debug_endpoints.nu
···
3
3
4
4
def main [] {
5
5
let did = "did:web:guestbook.gaze.systems"
6
-
let port = 3003
7
-
let debug_port = $port + 1
6
+
let port = resolve-test-port 3003
7
+
let debug_port = resolve-test-debug-port ($port + 1)
8
8
let url = $"http://localhost:($port)"
9
9
let debug_url = $"http://localhost:($debug_port)"
10
10
let db_path = (mktemp -d -t hydrant_debug_test.XXXXXX)
···
76
76
print "PASSED: /debug/iter on events returns JSON objects"
77
77
}
78
78
79
-
# 4. Test size in /stats
80
-
print "testing size in /stats"
79
+
# 4. Test sizes in /stats
80
+
print "testing sizes in /stats"
81
81
let stats = http get $"($url)/stats"
82
-
if ($stats.size | is-empty) {
83
-
print "FAILED: /stats returned empty size"
82
+
if ($stats.sizes | is-empty) {
83
+
print "FAILED: /stats returned empty sizes"
84
84
exit 1
85
85
}
86
-
if not ("repos" in ($stats.size | columns)) {
87
-
print "FAILED: /stats missing 'repos' in size"
86
+
if not ("repos" in ($stats.sizes | columns)) {
87
+
print "FAILED: /stats missing 'repos' in sizes"
88
88
exit 1
89
89
}
90
90
print "PASSED: /stats returns keyspace sizes"
+5
-5
tests/ephemeral_gc.nu
+5
-5
tests/ephemeral_gc.nu
···
2
2
use common.nu *
3
3
4
4
# start hydrant in ephemeral mode
5
-
def run-ephemeral-instance [name: string, scenario_closure: closure] {
6
-
let port = 3006
7
-
let debug_port = $port + 1
5
+
def run-ephemeral-instance [name: string, port: int, scenario_closure: closure] {
6
+
let debug_port = resolve-test-debug-port ($port + 1)
8
7
let url = $"http://localhost:($port)"
9
8
let debug_url = $"http://localhost:($debug_port)"
10
9
let db_path = (mktemp -d -t hydrant_ephemeral_gc_test.XXXXXX)
···
46
45
}
47
46
48
47
def main [] {
48
+
let port = resolve-test-port 3006
49
49
let repo1 = "did:web:guestbook.gaze.systems"
50
50
51
51
# verify TTL tick runs without error when no events are eligible for expiry
52
-
run-ephemeral-instance "TTL tick is safe with no eligible events" { |url, debug_url|
52
+
run-ephemeral-instance "TTL tick is safe with no eligible events" $port { |url, debug_url|
53
53
print $"adding repo ($repo1)..."
54
54
http put -t application/json $"($url)/repos" [{ did: ($repo1) }]
55
55
···
71
71
}
72
72
73
73
# plant a past watermark, trigger the real TTL path, and verify all events and blocks are gone
74
-
run-ephemeral-instance "TTL tick with past watermark deletes events and blocks" { |url, debug_url|
74
+
run-ephemeral-instance "TTL tick with past watermark deletes events and blocks" $port { |url, debug_url|
75
75
print $"adding repo ($repo1)..."
76
76
http put -t application/json $"($url)/repos" [{ did: ($repo1) }]
77
77
+3
-2
tests/repo_sync_integrity.nu
+3
-2
tests/repo_sync_integrity.nu
···
96
96
def main [] {
97
97
let did = "did:plc:dfl62fgb7wtjj3fcbb72naae"
98
98
let pds = "https://zwsp.xyz"
99
-
let port = 3001
99
+
let port = resolve-test-port 3001
100
100
let url = $"http://localhost:($port)"
101
-
let debug_url = $"http://127.0.0.1:($port + 1)"
101
+
let debug_port = resolve-test-debug-port ($port + 1)
102
+
let debug_url = $"http://127.0.0.1:($debug_port)"
102
103
let db_path = (mktemp -d -t hydrant_test.XXXXXX)
103
104
104
105
print $"testing backfill integrity for ($did)..."
+99
tests/repos_api.nu
+99
tests/repos_api.nu
···
1
+
#!/usr/bin/env nu
2
+
use common.nu *
3
+
4
+
def test-repos [url: string] {
5
+
print "verifying /repos pagination and filtering..."
6
+
7
+
# 1. test limit
8
+
print " testing limit=1..."
9
+
let items = (http get $"($url)/repos?limit=1" | from json -o)
10
+
if ($items | length) != 1 {
11
+
fail "expected 1 item with limit=1"
12
+
}
13
+
print $" count: ($items | length)"
14
+
15
+
# 2. test partition=all
16
+
print " testing partition=all..."
17
+
let all_items = (http get $"($url)/repos?partition=all" | from json -o)
18
+
print $" count: ($all_items | length)"
19
+
20
+
# 3. test cursor (if we have enough items)
21
+
if ($all_items | length) > 1 {
22
+
let first_did = ($all_items | get 0).did
23
+
print $" testing cursor with did ($first_did)..."
24
+
let cursor_items = (http get $"($url)/repos?cursor=($first_did)&limit=1" | from json -o)
25
+
if ($cursor_items | length) > 0 {
26
+
let next_did = ($cursor_items | get 0).did
27
+
if $first_did == $next_did {
28
+
fail "cursor did should be excluded from results"
29
+
}
30
+
print $" next did: ($next_did)"
31
+
}
32
+
}
33
+
34
+
# 4. test partition=pending
35
+
print " testing partition=pending..."
36
+
let pending_items = (http get $"($url)/repos?partition=pending" | from json -o)
37
+
print $" pending count: ($pending_items | length)"
38
+
39
+
# 5. test partition=resync
40
+
print " testing partition=resync..."
41
+
let resync_items = (http get $"($url)/repos?partition=resync" | from json -o)
42
+
print $" resync count: ($resync_items | length)"
43
+
44
+
print "all /repos pagination and filtering tests passed!"
45
+
}
46
+
47
+
def test-errors [url: string] {
48
+
print "verifying /repos error handling..."
49
+
50
+
# invalid DID in PUT
51
+
print " testing PUT /repos with invalid DID..."
52
+
http put -f -e -t application/json $"($url)/repos" { did: "invalid" }
53
+
| assert-status 400 "PUT /repos invalid DID"
54
+
55
+
# invalid DID in DELETE
56
+
print " testing DELETE /repos with invalid DID..."
57
+
http delete -f -e -t application/json $"($url)/repos" --data { did: "invalid" }
58
+
| assert-status 400 "DELETE /repos invalid DID"
59
+
60
+
# invalid cursor in GET
61
+
print " testing GET /repos with invalid cursor..."
62
+
http get -f -e $"($url)/repos?cursor=invalid"
63
+
| assert-status 400 "GET /repos invalid cursor"
64
+
65
+
# invalid partition in GET
66
+
print " testing GET /repos with invalid partition..."
67
+
http get -f -e $"($url)/repos?partition=invalid"
68
+
| assert-status 400 "GET /repos invalid partition"
69
+
70
+
print "all /repos error handling tests passed!"
71
+
}
72
+
73
+
def main [] {
74
+
let port = resolve-test-port 3001
75
+
let url = $"http://localhost:($port)"
76
+
let db_path = (mktemp -d -t hydrant_repos_api.XXXXXX)
77
+
78
+
print $"starting hydrant for repos API verification..."
79
+
let binary = build-hydrant
80
+
let instance = (with-env { HYDRANT_MODE: "filter" } {
81
+
start-hydrant $binary $db_path $port
82
+
})
83
+
84
+
if not (wait-for-api $url) {
85
+
fail "hydrant did not start" $instance.pid
86
+
}
87
+
88
+
# seed a couple of repos so pagination tests have data
89
+
let dids = [
90
+
"did:plc:dfl62fgb7wtjj3fcbb72naae"
91
+
"did:plc:q6gjnv26m4ay3m42ojvzx2m4"
92
+
]
93
+
http put -t application/json $"($url)/repos" ($dids | each { |d| { did: $d } })
94
+
95
+
test-repos $url
96
+
test-errors $url
97
+
98
+
kill $instance.pid
99
+
}
-141
tests/repos_api_test.nu
-141
tests/repos_api_test.nu
···
1
-
#!/usr/bin/env nu
2
-
3
-
def test-repos [url: string] {
4
-
print "verifying /repos pagination and filtering..."
5
-
6
-
# 1. Test limit
7
-
print " testing limit=1..."
8
-
let items = (http get $"($url)/repos?limit=1" | from json -o)
9
-
let count = ($items | length)
10
-
print $" count: ($count)"
11
-
if $count != 1 {
12
-
print " FAILED: expected 1 item"
13
-
exit 1
14
-
}
15
-
16
-
# 2. Test partition=all
17
-
print " testing partition=all..."
18
-
let all_items = (http get $"($url)/repos?partition=all" | from json -o)
19
-
print $" count: ($all_items | length)"
20
-
21
-
# 3. Test cursor (if we have items)
22
-
if ($all_items | length) > 1 {
23
-
let first_did = ($all_items | get 0).did
24
-
print $" testing cursor with did ($first_did)..."
25
-
let cursor_items = (http get $"($url)/repos?cursor=($first_did)&limit=1" | from json -o)
26
-
if ($cursor_items | length) > 0 {
27
-
let next_did = ($cursor_items | get 0).did
28
-
if $first_did == $next_did {
29
-
print " FAILED: cursor did should be excluded"
30
-
exit 1
31
-
}
32
-
print $" next did: ($next_did)"
33
-
}
34
-
}
35
-
36
-
# 4. Test partition=pending
37
-
print " testing partition=pending..."
38
-
let pending_items = (http get $"($url)/repos?partition=pending" | from json -o)
39
-
print $" pending count: ($pending_items | length)"
40
-
41
-
# 5. Test partition=resync
42
-
print " testing partition=resync..."
43
-
let resync_items = (http get $"($url)/repos?partition=resync" | from json -o)
44
-
print $" resync count: ($resync_items | length)"
45
-
46
-
print "all /repos pagination and filtering tests passed!"
47
-
}
48
-
49
-
def test-errors [url: string] {
50
-
print "verifying /repos error handling..."
51
-
52
-
# 1. Invalid DID in PUT
53
-
print " testing PUT /repos with invalid DID..."
54
-
let resp_put = (http put -f -e -t application/json $"($url)/repos" { did: "invalid" })
55
-
if $resp_put.status != 400 {
56
-
print $" FAILED: expected 400, got ($resp_put.status)"
57
-
exit 1
58
-
}
59
-
60
-
# 2. Invalid DID in DELETE
61
-
print " testing DELETE /repos with invalid DID..."
62
-
let resp_del = (http delete -f -e -t application/json $"($url)/repos" --data { did: "invalid" })
63
-
if $resp_del.status != 400 {
64
-
print $" FAILED: expected 400, got ($resp_del.status)"
65
-
exit 1
66
-
}
67
-
68
-
# 3. Invalid cursor in GET
69
-
print " testing GET /repos with invalid cursor..."
70
-
let resp_get_cursor = (http get -f -e $"($url)/repos?cursor=invalid")
71
-
if $resp_get_cursor.status != 400 {
72
-
print $" FAILED: expected 400, got ($resp_get_cursor.status)"
73
-
exit 1
74
-
}
75
-
76
-
# 4. Invalid partition in GET
77
-
print " testing GET /repos with invalid partition..."
78
-
let resp_get_part = (http get -f -e $"($url)/repos?partition=invalid")
79
-
if $resp_get_part.status != 400 {
80
-
print $" FAILED: expected 400, got ($resp_get_part.status)"
81
-
exit 1
82
-
}
83
-
84
-
print "all /repos error handling tests passed!"
85
-
}
86
-
87
-
def main [] {
88
-
let port = 3001
89
-
let url = $"http://localhost:($port)"
90
-
let db_path = (mktemp -d -t hydrant_api_test.XXXXXX)
91
-
92
-
print $"starting hydrant for API verification..."
93
-
let binary = (build-hydrant)
94
-
let instance = (start-hydrant $binary $db_path $port)
95
-
96
-
if (wait-for-api $url) {
97
-
# add a few repos
98
-
let dids = [
99
-
"did:plc:dfl62fgb7wtjj3fcbb72naae"
100
-
"did:plc:q6gjnv26m4ay3m42ojvzx2m4"
101
-
]
102
-
http put -t application/json $"($url)/repos" ($dids | each { |d| { did: $d } })
103
-
104
-
test-repos $url
105
-
test-errors $url
106
-
}
107
-
108
-
kill $instance.pid
109
-
}
110
-
111
-
# Helper to build hydrant
112
-
def build-hydrant [] {
113
-
cargo build --quiet
114
-
"./target/debug/hydrant"
115
-
}
116
-
117
-
# Helper to start hydrant
118
-
def start-hydrant [binary: string, db_path: string, port: int] {
119
-
let log_file = $"($db_path)/hydrant.log"
120
-
let pid = (with-env {
121
-
HYDRANT_DATABASE_PATH: $db_path,
122
-
HYDRANT_API_PORT: ($port | into string),
123
-
HYDRANT_DEBUG_PORT: (($port + 1) | into string),
124
-
HYDRANT_MODE: "filter",
125
-
HYDRANT_LOG_LEVEL: "info"
126
-
} {
127
-
sh -c $"($binary) >($log_file) 2>&1 & echo $!" | str trim | into int
128
-
})
129
-
{ pid: $pid, log: $log_file }
130
-
}
131
-
132
-
# Helper to wait for api
133
-
def wait-for-api [url: string] {
134
-
for i in 1..20 {
135
-
if (try { (http get $"($url)/health") == "OK" } catch { false }) {
136
-
return true
137
-
}
138
-
sleep 500ms
139
-
}
140
-
false
141
-
}
+107
tests/run_all.nu
+107
tests/run_all.nu
···
1
+
#!/usr/bin/env nu
2
+
# run all hydrant integration tests in parallel with automatically assigned free ports.
3
+
#
4
+
# usage:
5
+
# nu tests/run_all.nu
6
+
# nu tests/run_all.nu --only [stream repos_api]
7
+
8
+
def get_free_ports [count: int] {
9
+
mut chosen = []
10
+
loop {
11
+
let p = port
12
+
if ($chosen | any {$in == $p}) {
13
+
continue;
14
+
}
15
+
$chosen = $chosen | append $p
16
+
if ($chosen | length) == $count {
17
+
break
18
+
}
19
+
}
20
+
$chosen
21
+
}
22
+
23
+
def run-test [] {
24
+
let result = (with-env {
25
+
HYDRANT_API_PORT: $in.api
26
+
HYDRANT_DEBUG_PORT: $in.debug
27
+
HYDRANT_TEST_MOCK_PORT: $in.mock
28
+
HYDRANT_BINARY: "target/debug/hydrant"
29
+
} {
30
+
^nu $"tests/($in.name).nu" | complete
31
+
})
32
+
{
33
+
name: $in.name
34
+
success: ($result.exit_code == 0)
35
+
output: $result.stdout
36
+
stderr: $result.stderr
37
+
}
38
+
}
39
+
40
+
def main [--only: list<string> = []] {
41
+
print "building hydrant..."
42
+
# build defaults features
43
+
cargo build
44
+
# build backlinks
45
+
cargo build --features backlinks
46
+
print ""
47
+
48
+
# discover all test scripts, excluding infrastructure files
49
+
let excluded = ["common", "mock_relay", "run_all"]
50
+
let discovered = (
51
+
ls tests/*.nu
52
+
| get name
53
+
| each {path basename | str replace ".nu" ""}
54
+
| where {|name| not ($excluded | any {$in == $name})}
55
+
)
56
+
57
+
let tests = if ($only | is-empty) {
58
+
$discovered
59
+
} else {
60
+
$discovered | where {|t| $only | any {$in == $t}}
61
+
}
62
+
let ports = get_free_ports (($tests | length) * 3)
63
+
64
+
mut assigned = []
65
+
for test in ($tests | enumerate) {
66
+
let p = {($test | get index) * 3 + $in}
67
+
let entry = {
68
+
name: ($test | get item),
69
+
api: ($ports | get (0 | do $p)),
70
+
debug: ($ports | get (1 | do $p)),
71
+
mock: ($ports | get (2 | do $p))
72
+
}
73
+
$assigned = ($assigned | append $entry)
74
+
}
75
+
76
+
let groups = {
77
+
"authenticated_stream": "event_dependent",
78
+
"signal_filter": "event_dependent",
79
+
}
80
+
let grouped = $assigned | group-by {|t| $groups | get -o $t.name | default $t.name}
81
+
82
+
print $"running ($assigned | length) tests...\n"
83
+
84
+
let run_group = {each {timeit -o {run-test} | {time: $in.time, ...$in.output}}};
85
+
let results = $grouped | values | par-each {do $run_group} | flatten
86
+
87
+
print "\n=== results ===\n"
88
+
for r in $results {
89
+
if $r.success {
90
+
print $" PASSED ($r.name) in ($r.time)"
91
+
} else {
92
+
print $" FAILED ($r.name) in ($r.time)"
93
+
let combined = $"($r.output)\n($r.stderr)" | str trim
94
+
$combined | lines | each {print $" ($in)"}
95
+
print ""
96
+
}
97
+
}
98
+
99
+
let res = $results | group-by {$in.success}
100
+
let failed = $res | get -o false | default []
101
+
let passed = $res | get -o true | default []
102
+
print $"\n($passed | length) passed, ($failed | length) failed"
103
+
104
+
try { ^pkill "hydrant" }
105
+
106
+
if ($failed | length) > 0 { exit 1 }
107
+
}
+8
-3
tests/signal_filter_test.nu
tests/signal_filter.nu
+8
-3
tests/signal_filter_test.nu
tests/signal_filter.nu
···
7
7
let password = ($env_vars | get --optional TEST_PASSWORD)
8
8
9
9
if ($did | is-empty) or ($password | is-empty) {
10
-
print "error: TEST_REPO and TEST_PASSWORD must be set in .env"
11
-
exit 1
10
+
print "SKIP: TEST_REPO and TEST_PASSWORD not set in .env"
11
+
exit 0
12
12
}
13
13
14
-
let port = 3011
14
+
let port = resolve-test-port 3011
15
15
let url = $"http://localhost:($port)"
16
16
let db_path = (mktemp -d -t hydrant_signal_test.XXXXXX)
17
17
···
103
103
}
104
104
} else {
105
105
print "hydrant failed to start"
106
+
if ($instance.log | path exists) {
107
+
print "--- hydrant log ---"
108
+
print (open $instance.log)
109
+
print "-------------------"
110
+
}
106
111
}
107
112
108
113
print "stopping hydrant..."
+1
-1
tests/stream_test.nu
tests/stream.nu
+1
-1
tests/stream_test.nu
tests/stream.nu
+4
-8
tests/throttling_test.nu
tests/throttling.nu
+4
-8
tests/throttling_test.nu
tests/throttling.nu
···
9
9
}
10
10
11
11
# 2. setup ports and paths
12
-
let port = 3010
13
-
let mock_port = 3012
12
+
let port = resolve-test-port 3010
13
+
let mock_port = resolve-test-mock-port 3012
14
14
let url = $"http://localhost:($port)"
15
15
let mock_url = $"http://localhost:($mock_port)"
16
16
let db_path = (mktemp -d -t hydrant_throttling.XXXXXX)
···
26
26
| into int
27
27
)
28
28
print $"mock relay pid: ($mock_pid)"
29
-
30
-
# give mock relay a moment
31
-
sleep 1sec
32
29
33
30
# 4. start hydrant with low throttling limits
34
31
let binary = build-hydrant
···
86
83
87
84
# now check logs for throttling message
88
85
print "checking logs for throttling message..."
89
-
sleep 2sec # give logging a moment
90
86
91
87
let logs = (open $log_file | str replace --all "\n" " ")
92
88
if ($logs | str contains "throttling: above max pending") {
···
104
100
{"did": "did:web:mock4.com"}
105
101
]' $"($url)/repos"
106
102
107
-
print "waiting for crawler to wake up (max 10s)..."
108
-
sleep 15sec
103
+
print "waiting for crawler to wake up..."
104
+
sleep 1sec
109
105
110
106
# check logs for resumption message
111
107
let logs_after = (open $log_file | str replace --all "\n" " ")
+5
-4
tests/verify_crawler.nu
+5
-4
tests/verify_crawler.nu
···
9
9
}
10
10
11
11
# 2. setup ports and paths
12
-
let port = 3006
13
-
let mock_port = 3008
12
+
let port = resolve-test-port 3006
13
+
let mock_port = resolve-test-mock-port 3008
14
14
let url = $"http://localhost:($port)"
15
-
let debug_url = $"http://localhost:($port + 1)"
15
+
let debug_port = resolve-test-debug-port ($port + 1)
16
+
let debug_url = $"http://localhost:($debug_port)"
16
17
let mock_url = $"http://localhost:($mock_port)"
17
18
let db_path = (mktemp -d -t hydrant_full_net.XXXXXX)
18
19
···
46
47
HYDRANT_DISABLE_BACKFILL: "true",
47
48
HYDRANT_API_PORT: ($port | into string),
48
49
HYDRANT_ENABLE_DEBUG: "true", # for stats checking
49
-
HYDRANT_DEBUG_PORT: ($port + 1 | into string),
50
+
HYDRANT_DEBUG_PORT: (resolve-test-debug-port ($port + 1) | into string),
50
51
HYDRANT_LOG_LEVEL: "debug",
51
52
HYDRANT_CURSOR_SAVE_INTERVAL: "1" # faster save
52
53
} {