Ramjet is a relay consumer that supports configurable forward and track collections, as well as record reconciliation.
event-stream
relay
firehose
riblt
atprotocol
release: 0.1.1
+174
-8
+87
.github/workflows/release-binaries.yml
+87
.github/workflows/release-binaries.yml
···
1
+
name: Release binaries
2
+
3
+
on:
4
+
workflow_dispatch:
5
+
push:
6
+
branches:
7
+
- main
8
+
tags:
9
+
- "v*"
10
+
11
+
permissions:
12
+
contents: write
13
+
14
+
jobs:
15
+
build:
16
+
strategy:
17
+
matrix:
18
+
target:
19
+
- name: x86_64-unknown-linux-gnu
20
+
os: ubuntu-latest
21
+
suffix: linux-amd64
22
+
- name: aarch64-unknown-linux-gnu
23
+
os: ubuntu-latest
24
+
suffix: linux-arm64
25
+
cross: true
26
+
- name: x86_64-apple-darwin
27
+
os: macos-latest
28
+
suffix: darwin-amd64
29
+
- name: aarch64-apple-darwin
30
+
os: macos-latest
31
+
suffix: darwin-arm64
32
+
binary:
33
+
- name: ramjet
34
+
- name: rjtop
35
+
36
+
runs-on: ${{ matrix.target.os }}
37
+
38
+
steps:
39
+
- uses: actions/checkout@v4
40
+
41
+
- name: Install Rust toolchain
42
+
uses: dtolnay/rust-toolchain@stable
43
+
with:
44
+
targets: ${{ matrix.target.name }}
45
+
46
+
- uses: Swatinem/rust-cache@v2
47
+
with:
48
+
key: ${{ matrix.target.name }}-${{ matrix.binary.name }}
49
+
50
+
- name: Install cross
51
+
if: matrix.target.cross
52
+
run: cargo install cross --git https://github.com/cross-rs/cross
53
+
54
+
- name: Build
55
+
run: |
56
+
BUILD_CMD="cargo"
57
+
if [ "${{ matrix.target.cross }}" = "true" ]; then
58
+
BUILD_CMD="cross"
59
+
fi
60
+
$BUILD_CMD build --release --target ${{ matrix.target.name }} --bin ${{ matrix.binary.name }}
61
+
62
+
- name: Package binary
63
+
run: |
64
+
cp target/${{ matrix.target.name }}/release/${{ matrix.binary.name }} ${{ matrix.binary.name }}-${{ matrix.target.suffix }}
65
+
chmod +x ${{ matrix.binary.name }}-${{ matrix.target.suffix }}
66
+
67
+
- uses: actions/upload-artifact@v4
68
+
with:
69
+
name: ${{ matrix.binary.name }}-${{ matrix.target.suffix }}
70
+
path: ${{ matrix.binary.name }}-${{ matrix.target.suffix }}
71
+
72
+
release:
73
+
needs: build
74
+
runs-on: ubuntu-latest
75
+
if: startsWith(github.ref, 'refs/tags/')
76
+
77
+
steps:
78
+
- uses: actions/download-artifact@v4
79
+
with:
80
+
path: artifacts
81
+
merge-multiple: true
82
+
83
+
- name: Create release
84
+
uses: softprops/action-gh-release@v2
85
+
with:
86
+
files: artifacts/*
87
+
generate_release_notes: true
+16
CHANGELOG.md
+16
CHANGELOG.md
···
1
+
# Changelog
2
+
3
+
## 0.1.1 - 2026-03-15
4
+
5
+
### Added
6
+
- Desync detection, HTTP metrics, and backfill integration (#1)
7
+
- Zstd dictionary compression and hybrid disk-backed backfill (#2)
8
+
- RIBLT reconciliation, backfill CLI parameter, and consumer groups (#3)
9
+
- Multi-stage Dockerfile for all ramjet binaries (#4)
10
+
11
+
### Changed
12
+
- Moved RIBLT cache to dedicated keyspace (#5)
13
+
14
+
## 0.1.0
15
+
16
+
- Initial release
+1
-1
Cargo.lock
+1
-1
Cargo.lock
+1
-1
Cargo.toml
+1
-1
Cargo.toml
+57
Dockerfile
+57
Dockerfile
···
1
+
# Multi-stage build for ramjet
2
+
# Builds and installs all 7 binaries from the project
3
+
4
+
# Build stage - use 1.94 to match rust-version in Cargo.toml
5
+
FROM rust:1.94-slim-bookworm AS builder
6
+
7
+
# Install system dependencies needed for building
8
+
RUN apt-get update && apt-get install -y \
9
+
pkg-config \
10
+
libssl-dev \
11
+
&& rm -rf /var/lib/apt/lists/*
12
+
13
+
# Set working directory
14
+
WORKDIR /usr/src/app
15
+
16
+
# Copy the entire project
17
+
COPY . .
18
+
19
+
# Build all binaries in release mode
20
+
RUN cargo build --release --bins
21
+
22
+
# Runtime stage - use distroless for minimal attack surface
23
+
FROM gcr.io/distroless/cc-debian12
24
+
25
+
# Create directory for binaries
26
+
WORKDIR /usr/local/bin
27
+
28
+
# Copy all built binaries from builder stage
29
+
COPY --from=builder /usr/src/app/target/release/ramjet .
30
+
COPY --from=builder /usr/src/app/target/release/ramjet_consumer .
31
+
COPY --from=builder /usr/src/app/target/release/ramjet_data .
32
+
COPY --from=builder /usr/src/app/target/release/ramjet_dictgen .
33
+
COPY --from=builder /usr/src/app/target/release/ramjet_forecast .
34
+
COPY --from=builder /usr/src/app/target/release/ramjet_writer .
35
+
COPY --from=builder /usr/src/app/target/release/rjtop .
36
+
37
+
# Default to the main service binary
38
+
# Users can override with specific binary: docker run <image> ramjet_consumer --help
39
+
# docker run <image> ramjet --help
40
+
# docker run <image> ramjet_consumer --help
41
+
# docker run <image> ramjet_data --help
42
+
# docker run <image> ramjet_dictgen --help
43
+
# docker run <image> ramjet_forecast --help
44
+
# docker run <image> ramjet_writer --help
45
+
# docker run <image> rjtop --help
46
+
CMD ["ramjet", "--help"]
47
+
48
+
# Add labels for documentation
49
+
LABEL org.opencontainers.image.title="ramjet"
50
+
LABEL org.opencontainers.image.description="ATProtocol event-stream, record, and blob service built on fjall"
51
+
LABEL org.opencontainers.image.authors="Nick Gerakines <nick.gerakines@gmail.com>"
52
+
LABEL org.opencontainers.image.source="https://tangled.org/ngerakines.me/ramjet"
53
+
LABEL org.opencontainers.image.version="0.1.1"
54
+
LABEL org.opencontainers.image.licenses="MIT"
55
+
56
+
# Document available binaries
57
+
LABEL binaries="ramjet,ramjet_consumer,ramjet_data,ramjet_dictgen,ramjet_forecast,ramjet_writer,rjtop"
+1
-1
README.md
+1
-1
README.md
···
118
118
119
119
### Health and metrics
120
120
121
-
- `GET /_health` — returns `{"status":"ok","version":"0.1.0"}`
121
+
- `GET /_health` — returns `{"status":"ok","version":"0.1.1"}`
122
122
- `GET /metrics` — Prometheus text format (includes HTTP metrics, pipeline counters, tokio task metrics, fan-out queue depths)
123
123
124
124
### XRPC endpoints
+8
-5
src/server/reconciliation.rs
+8
-5
src/server/reconciliation.rs
···
100
100
let use_cache = params.collection.is_none();
101
101
if use_cache {
102
102
let cache_key = riblt_cache_key(did);
103
-
if let Ok(Some(cached)) = state.db.meta.get(cache_key.as_bytes()) {
103
+
if let Ok(Some(cached)) = state.db.riblt_cache.get(cache_key.as_bytes()) {
104
104
let slice: &[u8] = &cached;
105
105
if let Some((cached_rev, record_count, riblt_bytes)) = decode_cache_value(slice) {
106
106
if cached_rev == repo_state.rev {
···
126
126
if use_cache {
127
127
let cache_key = riblt_cache_key(did);
128
128
let cache_value = encode_cache_value(&rev, record_count, &riblt_bytes);
129
-
let _ = state.db.meta.insert(cache_key.as_bytes(), &cache_value);
129
+
let _ = state
130
+
.db
131
+
.riblt_cache
132
+
.insert(cache_key.as_bytes(), &cache_value);
130
133
}
131
134
riblt_response(&riblt_bytes, &rev, record_count)
132
135
}
···
258
261
/// Called when records change so the next request rebuilds the sketch.
259
262
pub fn invalidate_sketch_cache(db: &FjallDb, did: &str) {
260
263
let cache_key = riblt_cache_key(did);
261
-
let _ = db.meta.remove(cache_key.as_bytes());
264
+
let _ = db.riblt_cache.remove(cache_key.as_bytes());
262
265
}
263
266
264
267
// -- Cache encoding --
265
268
//
266
269
// Format: [2B rev_len LE][rev bytes][8B record_count LE][RibltFile bytes]
267
270
268
-
fn riblt_cache_key(did: &str) -> String {
269
-
format!("riblt\x00{did}")
271
+
fn riblt_cache_key(did: &str) -> &str {
272
+
did
270
273
}
271
274
272
275
fn encode_cache_value(rev: &str, record_count: u64, riblt_bytes: &[u8]) -> Vec<u8> {
+3
src/storage/mod.rs
+3
src/storage/mod.rs
···
40
40
pub blobs: fjall::Keyspace,
41
41
/// Blob metadata: `CID → ref count + size + path`.
42
42
pub blob_meta: fjall::Keyspace,
43
+
/// Cached RIBLT reconciliation sketches: `DID → rev + count + sketch`.
44
+
pub riblt_cache: fjall::Keyspace,
43
45
/// Optional zstd dictionary bytes for event compression.
44
46
zstd_dict: Option<Vec<u8>>,
45
47
/// Atomic event sequence counter shared between writer and backfill.
···
106
108
handle_to_did: ks("handle_to_did")?,
107
109
blobs: ks("blobs")?,
108
110
blob_meta: ks("blob_meta")?,
111
+
riblt_cache: ks("riblt_cache")?,
109
112
meta,
110
113
db,
111
114
zstd_dict,