Ramjet is a relay consumer that supports configurable forward and track collections, as well as record reconciliation.
event-stream relay firehose riblt atprotocol
13
fork

Configure Feed

Select the types of activity you want to include in your feed.

release: 0.1.1

+174 -8
+87
.github/workflows/release-binaries.yml
··· 1 + name: Release binaries 2 + 3 + on: 4 + workflow_dispatch: 5 + push: 6 + branches: 7 + - main 8 + tags: 9 + - "v*" 10 + 11 + permissions: 12 + contents: write 13 + 14 + jobs: 15 + build: 16 + strategy: 17 + matrix: 18 + target: 19 + - name: x86_64-unknown-linux-gnu 20 + os: ubuntu-latest 21 + suffix: linux-amd64 22 + - name: aarch64-unknown-linux-gnu 23 + os: ubuntu-latest 24 + suffix: linux-arm64 25 + cross: true 26 + - name: x86_64-apple-darwin 27 + os: macos-latest 28 + suffix: darwin-amd64 29 + - name: aarch64-apple-darwin 30 + os: macos-latest 31 + suffix: darwin-arm64 32 + binary: 33 + - name: ramjet 34 + - name: rjtop 35 + 36 + runs-on: ${{ matrix.target.os }} 37 + 38 + steps: 39 + - uses: actions/checkout@v4 40 + 41 + - name: Install Rust toolchain 42 + uses: dtolnay/rust-toolchain@stable 43 + with: 44 + targets: ${{ matrix.target.name }} 45 + 46 + - uses: Swatinem/rust-cache@v2 47 + with: 48 + key: ${{ matrix.target.name }}-${{ matrix.binary.name }} 49 + 50 + - name: Install cross 51 + if: matrix.target.cross 52 + run: cargo install cross --git https://github.com/cross-rs/cross 53 + 54 + - name: Build 55 + run: | 56 + BUILD_CMD="cargo" 57 + if [ "${{ matrix.target.cross }}" = "true" ]; then 58 + BUILD_CMD="cross" 59 + fi 60 + $BUILD_CMD build --release --target ${{ matrix.target.name }} --bin ${{ matrix.binary.name }} 61 + 62 + - name: Package binary 63 + run: | 64 + cp target/${{ matrix.target.name }}/release/${{ matrix.binary.name }} ${{ matrix.binary.name }}-${{ matrix.target.suffix }} 65 + chmod +x ${{ matrix.binary.name }}-${{ matrix.target.suffix }} 66 + 67 + - uses: actions/upload-artifact@v4 68 + with: 69 + name: ${{ matrix.binary.name }}-${{ matrix.target.suffix }} 70 + path: ${{ matrix.binary.name }}-${{ matrix.target.suffix }} 71 + 72 + release: 73 + needs: build 74 + runs-on: ubuntu-latest 75 + if: startsWith(github.ref, 'refs/tags/') 76 + 77 + steps: 78 + - uses: actions/download-artifact@v4 79 + with: 80 + path: artifacts 81 + merge-multiple: true 82 + 83 + - name: Create release 84 + uses: softprops/action-gh-release@v2 85 + with: 86 + files: artifacts/* 87 + generate_release_notes: true
+16
CHANGELOG.md
··· 1 + # Changelog 2 + 3 + ## 0.1.1 - 2026-03-15 4 + 5 + ### Added 6 + - Desync detection, HTTP metrics, and backfill integration (#1) 7 + - Zstd dictionary compression and hybrid disk-backed backfill (#2) 8 + - RIBLT reconciliation, backfill CLI parameter, and consumer groups (#3) 9 + - Multi-stage Dockerfile for all ramjet binaries (#4) 10 + 11 + ### Changed 12 + - Moved RIBLT cache to dedicated keyspace (#5) 13 + 14 + ## 0.1.0 15 + 16 + - Initial release
+1 -1
Cargo.lock
··· 2524 2524 2525 2525 [[package]] 2526 2526 name = "ramjet" 2527 - version = "0.1.0" 2527 + version = "0.1.1" 2528 2528 dependencies = [ 2529 2529 "anyhow", 2530 2530 "atproto-dasl",
+1 -1
Cargo.toml
··· 1 1 [package] 2 2 name = "ramjet" 3 - version = "0.1.0" 3 + version = "0.1.1" 4 4 edition = "2024" 5 5 rust-version = "1.94" 6 6 description = "ATProtocol event-stream, record, and blob service built on fjall"
+57
Dockerfile
··· 1 + # Multi-stage build for ramjet 2 + # Builds and installs all 7 binaries from the project 3 + 4 + # Build stage - use 1.94 to match rust-version in Cargo.toml 5 + FROM rust:1.94-slim-bookworm AS builder 6 + 7 + # Install system dependencies needed for building 8 + RUN apt-get update && apt-get install -y \ 9 + pkg-config \ 10 + libssl-dev \ 11 + && rm -rf /var/lib/apt/lists/* 12 + 13 + # Set working directory 14 + WORKDIR /usr/src/app 15 + 16 + # Copy the entire project 17 + COPY . . 18 + 19 + # Build all binaries in release mode 20 + RUN cargo build --release --bins 21 + 22 + # Runtime stage - use distroless for minimal attack surface 23 + FROM gcr.io/distroless/cc-debian12 24 + 25 + # Create directory for binaries 26 + WORKDIR /usr/local/bin 27 + 28 + # Copy all built binaries from builder stage 29 + COPY --from=builder /usr/src/app/target/release/ramjet . 30 + COPY --from=builder /usr/src/app/target/release/ramjet_consumer . 31 + COPY --from=builder /usr/src/app/target/release/ramjet_data . 32 + COPY --from=builder /usr/src/app/target/release/ramjet_dictgen . 33 + COPY --from=builder /usr/src/app/target/release/ramjet_forecast . 34 + COPY --from=builder /usr/src/app/target/release/ramjet_writer . 35 + COPY --from=builder /usr/src/app/target/release/rjtop . 36 + 37 + # Default to the main service binary 38 + # Users can override with specific binary: docker run <image> ramjet_consumer --help 39 + # docker run <image> ramjet --help 40 + # docker run <image> ramjet_consumer --help 41 + # docker run <image> ramjet_data --help 42 + # docker run <image> ramjet_dictgen --help 43 + # docker run <image> ramjet_forecast --help 44 + # docker run <image> ramjet_writer --help 45 + # docker run <image> rjtop --help 46 + CMD ["ramjet", "--help"] 47 + 48 + # Add labels for documentation 49 + LABEL org.opencontainers.image.title="ramjet" 50 + LABEL org.opencontainers.image.description="ATProtocol event-stream, record, and blob service built on fjall" 51 + LABEL org.opencontainers.image.authors="Nick Gerakines <nick.gerakines@gmail.com>" 52 + LABEL org.opencontainers.image.source="https://tangled.org/ngerakines.me/ramjet" 53 + LABEL org.opencontainers.image.version="0.1.1" 54 + LABEL org.opencontainers.image.licenses="MIT" 55 + 56 + # Document available binaries 57 + LABEL binaries="ramjet,ramjet_consumer,ramjet_data,ramjet_dictgen,ramjet_forecast,ramjet_writer,rjtop"
+1 -1
README.md
··· 118 118 119 119 ### Health and metrics 120 120 121 - - `GET /_health` — returns `{"status":"ok","version":"0.1.0"}` 121 + - `GET /_health` — returns `{"status":"ok","version":"0.1.1"}` 122 122 - `GET /metrics` — Prometheus text format (includes HTTP metrics, pipeline counters, tokio task metrics, fan-out queue depths) 123 123 124 124 ### XRPC endpoints
+8 -5
src/server/reconciliation.rs
··· 100 100 let use_cache = params.collection.is_none(); 101 101 if use_cache { 102 102 let cache_key = riblt_cache_key(did); 103 - if let Ok(Some(cached)) = state.db.meta.get(cache_key.as_bytes()) { 103 + if let Ok(Some(cached)) = state.db.riblt_cache.get(cache_key.as_bytes()) { 104 104 let slice: &[u8] = &cached; 105 105 if let Some((cached_rev, record_count, riblt_bytes)) = decode_cache_value(slice) { 106 106 if cached_rev == repo_state.rev { ··· 126 126 if use_cache { 127 127 let cache_key = riblt_cache_key(did); 128 128 let cache_value = encode_cache_value(&rev, record_count, &riblt_bytes); 129 - let _ = state.db.meta.insert(cache_key.as_bytes(), &cache_value); 129 + let _ = state 130 + .db 131 + .riblt_cache 132 + .insert(cache_key.as_bytes(), &cache_value); 130 133 } 131 134 riblt_response(&riblt_bytes, &rev, record_count) 132 135 } ··· 258 261 /// Called when records change so the next request rebuilds the sketch. 259 262 pub fn invalidate_sketch_cache(db: &FjallDb, did: &str) { 260 263 let cache_key = riblt_cache_key(did); 261 - let _ = db.meta.remove(cache_key.as_bytes()); 264 + let _ = db.riblt_cache.remove(cache_key.as_bytes()); 262 265 } 263 266 264 267 // -- Cache encoding -- 265 268 // 266 269 // Format: [2B rev_len LE][rev bytes][8B record_count LE][RibltFile bytes] 267 270 268 - fn riblt_cache_key(did: &str) -> String { 269 - format!("riblt\x00{did}") 271 + fn riblt_cache_key(did: &str) -> &str { 272 + did 270 273 } 271 274 272 275 fn encode_cache_value(rev: &str, record_count: u64, riblt_bytes: &[u8]) -> Vec<u8> {
+3
src/storage/mod.rs
··· 40 40 pub blobs: fjall::Keyspace, 41 41 /// Blob metadata: `CID → ref count + size + path`. 42 42 pub blob_meta: fjall::Keyspace, 43 + /// Cached RIBLT reconciliation sketches: `DID → rev + count + sketch`. 44 + pub riblt_cache: fjall::Keyspace, 43 45 /// Optional zstd dictionary bytes for event compression. 44 46 zstd_dict: Option<Vec<u8>>, 45 47 /// Atomic event sequence counter shared between writer and backfill. ··· 106 108 handle_to_did: ks("handle_to_did")?, 107 109 blobs: ks("blobs")?, 108 110 blob_meta: ks("blob_meta")?, 111 + riblt_cache: ks("riblt_cache")?, 109 112 meta, 110 113 db, 111 114 zstd_dict,