See the best posts from any Bluesky account
Add firehose-worker service and rebalance memory limits
- Trim queue-worker memory from 1G to 512M; threshold-scan and backfill
are I/O bound so 1G was always generous.
- Add firehose-worker (384M / 0.5 CPU) mirroring jetstream-worker, running
node ace.js firehose:watch. The command doesn't exist yet — Phase 2 will
land it. The service will fail to start until then; that's fine.
- Bind-mount clickhouse/config.d/memory.xml into the ClickHouse container
so the internal memory caps actually apply.
- Thread FIREHOSE_WEBHOOK_URL_1K / _10K / FIREHOSE_JETSTREAM_URL through
common-env.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+33
-1
+33
-1
docker-compose.yml
+33
-1
docker-compose.yml
···
17
17
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
18
18
BACKFILL_MAX_POSTS: ${BACKFILL_MAX_POSTS:-10000}
19
19
OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS}
20
+
FIREHOSE_WEBHOOK_URL_1K: ${FIREHOSE_WEBHOOK_URL_1K:-}
21
+
FIREHOSE_WEBHOOK_URL_10K: ${FIREHOSE_WEBHOOK_URL_10K:-}
22
+
FIREHOSE_JETSTREAM_URL: ${FIREHOSE_JETSTREAM_URL:-}
20
23
21
24
x-logging: &default-logging
22
25
driver: json-file
···
31
34
- '8123:8123'
32
35
volumes:
33
36
- clickhouse-data:/var/lib/clickhouse
37
+
- ./clickhouse/config.d/memory.xml:/etc/clickhouse-server/config.d/memory.xml:ro
34
38
environment:
35
39
CLICKHOUSE_DB: favs
36
40
CLICKHOUSE_USER: favs
···
147
151
failure_action: rollback
148
152
resources:
149
153
limits:
150
-
memory: 1G
154
+
memory: 512M
151
155
cpus: '0.5'
152
156
logging: *default-logging
153
157
healthcheck:
···
159
163
environment:
160
164
<<: *common-env
161
165
OTEL_SERVICE_NAME: favs-blue-worker
166
+
167
+
firehose-worker:
168
+
image: favs-blue:${GIT_SHA:-latest}
169
+
command: node ace.js firehose:watch
170
+
stop_grace_period: 30s
171
+
volumes:
172
+
- sqlite-data:/data
173
+
deploy:
174
+
restart_policy:
175
+
condition: any
176
+
update_config:
177
+
order: stop-first
178
+
failure_action: rollback
179
+
resources:
180
+
limits:
181
+
memory: 384M
182
+
cpus: '0.5'
183
+
logging: *default-logging
184
+
healthcheck:
185
+
test: ['CMD', 'node', '-e', 'process.exit(0)']
186
+
interval: 10s
187
+
timeout: 3s
188
+
retries: 3
189
+
start_period: 10s
190
+
environment:
191
+
<<: *common-env
192
+
JETSTREAM_URL: wss://jetstream2.us-east.bsky.network/subscribe
193
+
OTEL_SERVICE_NAME: favs-blue-firehose
162
194
163
195
volumes:
164
196
clickhouse-data: