an efficient binary archive format
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

remove c code

zach d4829551 eb520ff7

+301 -624
+12 -1
Makefile
··· 1 + lib: 2 + cargo build --release 3 + cp target/release/libbindle_file.a . 4 + cp target/release/libbindle_file.so libbindle.so || cp target/release/libbindle_file.dylib libbindle.dylib 5 + 6 + install: 7 + cp include/bindle.h /usr/local/include/bindle.h 8 + cp libbbindle.* /usr/local/lib 9 + 10 + uninstall: 11 + rm -f /usr/local/include/bindle.h /usr/local/lib/libbindle.* 12 + 1 13 test: 2 14 cargo test 3 - make -C c test
+1 -4
README.md
··· 1 1 # bindle-file 2 2 3 - `bindle` is a general purpose, multi-file archive format with support for zstd and direct reads for uncompressed values. 3 + `bindle` is a general purpose, binary archive format designed for efficient reads and writes. 4 4 5 - This repository contains `bindle-file` for Rust, which can also be used to build C libraries in 6 - `target/release/libbindle_file.a` and `target/release/libbindle_file.so`. `c/bindle.c` can also 7 - be used as a drop-in replacement for the Rust implementation in C projects. 8 5
-8
c/Makefile
··· 1 - .PHONY: test 2 - test: check 3 - python3 test.py a.bindl 4 - clang -Wall -Wextra -Wpedantic -I ../include -o bindle_c bindle-cli.c bindle.c `pkg-config --cflags --libs libzstd` 5 - 6 - check: 7 - clang -fsanitize=address -Wall -Wextra -Wpedantic -I ../include -o bindle_c bindle-cli.c bindle.c `pkg-config --cflags --libs libzstd` 8 -
-94
c/bindle-cli.c
··· 1 - #include "bindle.h" 2 - #include <stdio.h> 3 - #include <stdlib.h> 4 - #include <string.h> 5 - 6 - void print_usage(const char *prog) { 7 - printf("Usage: %s <command> <bindle_file> [args]\n", prog); 8 - printf("Commands:\n"); 9 - printf(" list List all entries\n"); 10 - printf(" cat <name> Output entry content to stdout\n"); 11 - printf(" add <name> <file> Add a single file to the archive\n"); 12 - printf(" pack <src_dir> Pack a directory into the archive\n"); 13 - printf(" unpack <dest_dir> Unpack archive to a directory\n"); 14 - printf(" vacuum Reclaim space from shadowed entries\n"); 15 - } 16 - 17 - int main(int argc, char **argv) { 18 - if (argc < 3) { 19 - print_usage(argv[0]); 20 - return 1; 21 - } 22 - 23 - // Switched back: command first, then bindle_file 24 - const char *cmd = argv[1]; 25 - const char *db_path = argv[2]; 26 - 27 - Bindle *b = bindle_open(db_path); 28 - if (!b) { 29 - fprintf(stderr, "Error: Could not open bindle '%s'\n", db_path); 30 - return 1; 31 - } 32 - 33 - if (strcmp(cmd, "list") == 0) { 34 - uint64_t count = bindle_length(b); 35 - printf("%-30s\n", "NAME"); 36 - printf("------------------------------\n"); 37 - for (uint64_t i = 0; i < count; i++) { 38 - size_t namelen = 0; 39 - const char *name = bindle_entry_name(b, i, &namelen); 40 - printf("%-30s\n", name); 41 - } 42 - } else if (strcmp(cmd, "cat") == 0) { 43 - if (argc < 4) { 44 - fprintf(stderr, "Usage: %s cat <file> <name>\n", argv[0]); 45 - bindle_close(b); 46 - return 1; 47 - } 48 - size_t out_len = 0; 49 - uint8_t *data = bindle_read(b, argv[3], &out_len); 50 - if (data) { 51 - fwrite(data, 1, out_len, stdout); 52 - free(data); 53 - } 54 - } else if (strcmp(cmd, "add") == 0) { 55 - if (argc < 5) { 56 - fprintf(stderr, "Usage: %s add <file> <name> <src_path>\n", argv[0]); 57 - bindle_close(b); 58 - return 1; 59 - } 60 - FILE *inf = fopen(argv[4], "rb"); 61 - if (inf) { 62 - fseek(inf, 0, SEEK_END); 63 - size_t len = ftell(inf); 64 - fseek(inf, 0, SEEK_SET); 65 - uint8_t *buf = malloc(len); 66 - fread(buf, 1, len, inf); 67 - fclose(inf); 68 - 69 - if (bindle_add(b, argv[3], buf, len, true)) { 70 - bindle_save(b); 71 - } 72 - free(buf); 73 - } 74 - } else if (strcmp(cmd, "pack") == 0) { 75 - if (argc < 4) { 76 - fprintf(stderr, "Usage: %s pack <file> <src_dir>\n", argv[0]); 77 - bindle_close(b); 78 - return 1; 79 - } 80 - bindle_pack(b, argv[3], true); 81 - } else if (strcmp(cmd, "unpack") == 0) { 82 - if (argc < 4) { 83 - fprintf(stderr, "Usage: %s unpack <file> <dest_dir>\n", argv[0]); 84 - bindle_close(b); 85 - return 1; 86 - } 87 - bindle_unpack(b, argv[3]); 88 - } else if (strcmp(cmd, "vacuum") == 0) { 89 - bindle_vacuum(b); 90 - } 91 - 92 - bindle_close(b); 93 - return 0; 94 - }
-438
c/bindle.c
··· 1 - #include "bindle.h" 2 - 3 - #include <dirent.h> 4 - #include <errno.h> 5 - #include <limits.h> 6 - #include <stdio.h> 7 - #include <stdlib.h> 8 - #include <string.h> 9 - #include <sys/file.h> 10 - #include <sys/stat.h> 11 - #include <zstd.h> 12 - 13 - #define BNDL_MAGIC "BINDL001" 14 - #define BNDL_ALIGN 8 15 - #define ALIGN_UP(n, m) (((n) + (m) - 1) & ~((m) - 1)) 16 - 17 - /* --- Private Disk Structures --- */ 18 - #pragma pack(push, 1) 19 - typedef struct { 20 - uint64_t offset; 21 - uint64_t compressed_size; 22 - uint64_t uncompressed_size; 23 - uint32_t crc32; 24 - uint16_t name_len; 25 - uint8_t compression_type; 26 - uint8_t _reserved; 27 - } BindleEntryRaw; 28 - 29 - typedef struct { 30 - uint64_t index_offset; 31 - uint64_t entry_count; 32 - } BindleFooterRaw; 33 - #pragma pack(pop) 34 - 35 - /* --- Private In-Memory Structures --- */ 36 - typedef struct { 37 - BindleEntryRaw meta; 38 - char *name; 39 - } BindleEntry; 40 - 41 - struct Bindle { 42 - char *path; 43 - FILE *fp; 44 - BindleEntry *entries; 45 - uint64_t count; 46 - uint64_t data_end; 47 - }; 48 - 49 - /* --- API Implementation --- */ 50 - 51 - Bindle *bindle_open(const char *path) { 52 - FILE *fp = fopen(path, "r+b"); 53 - if (!fp) { 54 - fp = fopen(path, "w+b"); 55 - } 56 - if (!fp) 57 - return NULL; 58 - 59 - flock(fileno(fp), LOCK_SH); 60 - 61 - Bindle *b = calloc(1, sizeof(Bindle)); 62 - b->path = strdup(path); 63 - b->fp = fp; 64 - 65 - fseek(fp, 0, SEEK_END); 66 - long file_size = ftell(fp); 67 - 68 - if (file_size == 0) { 69 - fwrite(BNDL_MAGIC, 8, 1, fp); 70 - b->data_end = 8; 71 - return b; 72 - } 73 - 74 - // Header check 75 - char magic[8]; 76 - fseek(fp, 0, SEEK_SET); 77 - if (fread(magic, 8, 1, fp) != 1 || memcmp(magic, BNDL_MAGIC, 8) != 0) { 78 - bindle_close(b); 79 - return NULL; 80 - } 81 - 82 - // Footer parse 83 - BindleFooterRaw footer; 84 - fseek(fp, file_size - sizeof(BindleFooterRaw), SEEK_SET); 85 - if (fread(&footer, sizeof(BindleFooterRaw), 1, fp) != 1) { 86 - bindle_close(b); 87 - return NULL; 88 - } 89 - 90 - b->count = footer.entry_count; 91 - b->data_end = footer.index_offset; 92 - b->entries = malloc(sizeof(BindleEntry) * b->count); 93 - 94 - // Index parse 95 - fseek(fp, footer.index_offset, SEEK_SET); 96 - for (uint64_t i = 0; i < b->count; i++) { 97 - fread(&b->entries[i].meta, sizeof(BindleEntryRaw), 1, fp); 98 - b->entries[i].name = malloc(b->entries[i].meta.name_len + 1); 99 - fread(b->entries[i].name, b->entries[i].meta.name_len, 1, fp); 100 - b->entries[i].name[b->entries[i].meta.name_len] = '\0'; 101 - 102 - size_t consumed = sizeof(BindleEntryRaw) + b->entries[i].meta.name_len; 103 - fseek(fp, ALIGN_UP(consumed, BNDL_ALIGN) - consumed, SEEK_CUR); 104 - } 105 - return b; 106 - } 107 - 108 - bool bindle_add(Bindle *b, const char *name, const uint8_t *data, size_t len, 109 - BindleCompress compress) { 110 - if (!b || !name) 111 - return false; 112 - 113 - size_t c_size = len; 114 - void *write_ptr = (void *)data; 115 - void *comp_buf = NULL; 116 - 117 - if (compress == BindleCompressZstd) { 118 - size_t bound = ZSTD_compressBound(len); 119 - comp_buf = malloc(bound); 120 - c_size = ZSTD_compress(comp_buf, bound, data, len, 3); 121 - if (ZSTD_isError(c_size)) { 122 - free(comp_buf); 123 - return false; 124 - } 125 - write_ptr = comp_buf; 126 - } 127 - 128 - // 1. Write data at current data_end 129 - fseek(b->fp, b->data_end, SEEK_SET); 130 - uint64_t offset = ftell(b->fp); 131 - fwrite(write_ptr, 1, c_size, b->fp); 132 - 133 - // 2. Align data_end 134 - size_t pad = ALIGN_UP(c_size, BNDL_ALIGN) - c_size; 135 - if (pad > 0) { 136 - uint8_t zero[8] = {0}; 137 - fwrite(zero, 1, pad, b->fp); 138 - } 139 - b->data_end = ftell(b->fp); 140 - 141 - // 3. Shadowing: Check if name already exists 142 - for (uint64_t i = 0; i < b->count; i++) { 143 - if (strcmp(b->entries[i].name, name) == 0) { 144 - b->entries[i].meta.offset = offset; 145 - b->entries[i].meta.compressed_size = c_size; 146 - b->entries[i].meta.uncompressed_size = len; 147 - b->entries[i].meta.compression_type = compress ? 1 : 0; 148 - if (comp_buf) 149 - free(comp_buf); 150 - return true; 151 - } 152 - } 153 - 154 - // 4. New Entry 155 - b->entries = realloc(b->entries, sizeof(BindleEntry) * (b->count + 1)); 156 - BindleEntry *e = &b->entries[b->count++]; 157 - e->name = strdup(name); 158 - e->meta = (BindleEntryRaw){offset, c_size, len, 0, (uint16_t)strlen(name), 159 - compress, 0}; 160 - 161 - if (comp_buf) 162 - free(comp_buf); 163 - return true; 164 - } 165 - 166 - uint8_t *bindle_read(Bindle *b, const char *name, size_t *out_len) { 167 - for (uint64_t i = 0; i < b->count; i++) { 168 - if (strcmp(b->entries[i].name, name) == 0) { 169 - BindleEntryRaw *m = &b->entries[i].meta; 170 - uint8_t *c_buf = malloc(m->compressed_size); 171 - fseek(b->fp, m->offset, SEEK_SET); 172 - fread(c_buf, 1, m->compressed_size, b->fp); 173 - 174 - if (m->compression_type == BindleCompressZstd) { 175 - uint8_t *u_buf = malloc(m->uncompressed_size); 176 - size_t actual = ZSTD_decompress(u_buf, m->uncompressed_size, c_buf, 177 - m->compressed_size); 178 - free(c_buf); 179 - *out_len = actual; 180 - return u_buf; 181 - } 182 - *out_len = m->compressed_size; 183 - return c_buf; 184 - } 185 - } 186 - return NULL; 187 - } 188 - 189 - const uint8_t *bindle_read_uncompressed_direct(Bindle *b, const char *name, 190 - size_t *out_len) { 191 - for (uint64_t i = 0; i < b->count; i++) { 192 - if (strcmp(b->entries[i].name, name) == 0) { 193 - BindleEntryRaw *m = &b->entries[i].meta; 194 - if (m->compression_type != BindleCompressNone) 195 - return NULL; 196 - 197 - uint8_t *buf = malloc(m->uncompressed_size); 198 - fseek(b->fp, m->offset, SEEK_SET); 199 - fread(buf, 1, m->uncompressed_size, b->fp); 200 - *out_len = m->uncompressed_size; 201 - return buf; 202 - } 203 - } 204 - return NULL; 205 - } 206 - 207 - bool bindle_save(Bindle *b) { 208 - flock(fileno(b->fp), LOCK_EX); 209 - fseek(b->fp, b->data_end, SEEK_SET); 210 - uint64_t index_start = b->data_end; 211 - 212 - for (uint64_t i = 0; i < b->count; i++) { 213 - fwrite(&b->entries[i].meta, sizeof(BindleEntryRaw), 1, b->fp); 214 - fwrite(b->entries[i].name, 1, b->entries[i].meta.name_len, b->fp); 215 - size_t consumed = sizeof(BindleEntryRaw) + b->entries[i].meta.name_len; 216 - size_t pad = ALIGN_UP(consumed, BNDL_ALIGN) - consumed; 217 - if (pad > 0) { 218 - uint8_t zero[8] = {0}; 219 - fwrite(zero, 1, pad, b->fp); 220 - } 221 - } 222 - 223 - BindleFooterRaw footer = {index_start, b->count}; 224 - fwrite(&footer, sizeof(BindleFooterRaw), 1, b->fp); 225 - fflush(b->fp); 226 - flock(fileno(b->fp), LOCK_SH); 227 - return true; 228 - } 229 - 230 - size_t bindle_length(const Bindle *b) { return b ? b->count : 0; } 231 - 232 - const char *bindle_entry_name(const Bindle *b, size_t index, size_t *namelen) { 233 - if (!b || index >= b->count) 234 - return NULL; 235 - *namelen = b->entries[index].meta.name_len; 236 - return b->entries[index].name; 237 - } 238 - 239 - void bindle_free_buffer(uint8_t *ptr) { free(ptr); } 240 - 241 - void bindle_close(Bindle *b) { 242 - if (!b) 243 - return; 244 - flock(fileno(b->fp), LOCK_UN); 245 - for (uint64_t i = 0; i < b->count; i++) 246 - free(b->entries[i].name); 247 - free(b->entries); 248 - fclose(b->fp); 249 - free(b->path); 250 - free(b); 251 - } 252 - 253 - bool bindle_vacuum(Bindle *b) { 254 - if (!b) 255 - return false; 256 - 257 - char tmp_path[1024]; 258 - snprintf(tmp_path, sizeof(tmp_path), "%s.tmp", b->path); 259 - FILE *out = fopen(tmp_path, "wb"); 260 - if (!out) 261 - return false; 262 - 263 - // 1. Write Header 264 - fwrite(BNDL_MAGIC, 8, 1, out); 265 - uint64_t current_offset = 8; 266 - 267 - // 2. Copy Live Data to Temp File 268 - for (uint64_t i = 0; i < b->count; i++) { 269 - uint64_t size = b->entries[i].meta.compressed_size; 270 - uint8_t *buf = malloc(size); 271 - 272 - fseek(b->fp, b->entries[i].meta.offset, SEEK_SET); 273 - fread(buf, 1, size, b->fp); 274 - 275 - fseek(out, current_offset, SEEK_SET); 276 - fwrite(buf, 1, size, out); 277 - 278 - // Update the in-memory metadata with the new offset 279 - b->entries[i].meta.offset = current_offset; 280 - 281 - size_t pad = ALIGN_UP(size, BNDL_ALIGN) - size; 282 - if (pad > 0) { 283 - uint8_t zero[8] = {0}; 284 - fwrite(zero, 1, pad, out); 285 - } 286 - current_offset += size + pad; 287 - free(buf); 288 - } 289 - 290 - // 3. Write Index and Footer to the Temp File (Matching Rust .save() logic) 291 - uint64_t index_start = current_offset; 292 - for (uint64_t i = 0; i < b->count; i++) { 293 - fwrite(&b->entries[i].meta, sizeof(BindleEntryRaw), 1, out); 294 - fwrite(b->entries[i].name, 1, b->entries[i].meta.name_len, out); 295 - 296 - size_t consumed = sizeof(BindleEntryRaw) + b->entries[i].meta.name_len; 297 - size_t pad = ALIGN_UP(consumed, BNDL_ALIGN) - consumed; 298 - if (pad > 0) { 299 - uint8_t zero[8] = {0}; 300 - fwrite(zero, 1, pad, out); 301 - } 302 - } 303 - 304 - BindleFooterRaw footer = {index_start, b->count}; 305 - fwrite(&footer, sizeof(BindleFooterRaw), 1, out); 306 - 307 - // 4. CRITICAL: Close and Unlock handles before Rename 308 - fflush(out); 309 - fclose(out); // Close the temp file handle 310 - 311 - flock(fileno(b->fp), LOCK_UN); // Explicitly unlock the original file 312 - fclose(b->fp); // Close the original file handle 313 - b->fp = NULL; 314 - 315 - // 5. Atomic Rename 316 - if (rename(tmp_path, b->path) != 0) { 317 - // If rename fails, we are in a bad state; attempt to re-open original 318 - b->fp = fopen(b->path, "r+b"); 319 - return false; 320 - } 321 - 322 - // 6. Re-open the new primary file 323 - b->fp = fopen(b->path, "r+b"); 324 - if (!b->fp) 325 - return false; 326 - 327 - flock(fileno(b->fp), LOCK_SH); 328 - b->data_end = index_start; 329 - 330 - return true; 331 - } 332 - 333 - /* --- Helper: Recursive directory creation (mkdir -p) --- */ 334 - static void ensure_dir_exists(const char *path) { 335 - char tmp[1024]; 336 - char *p = NULL; 337 - size_t len; 338 - 339 - snprintf(tmp, sizeof(tmp), "%s", path); 340 - len = strlen(tmp); 341 - if (tmp[len - 1] == '/') 342 - tmp[len - 1] = 0; 343 - 344 - for (p = tmp + 1; *p; p++) { 345 - if (*p == '/') { 346 - *p = 0; 347 - mkdir(tmp, 0755); 348 - *p = '/'; 349 - } 350 - } 351 - } 352 - 353 - /* --- Recursive Pack Implementation --- */ 354 - static bool pack_recursive(Bindle *b, const char *base_path, 355 - const char *current_path, bool compress) { 356 - DIR *dir = opendir(current_path); 357 - if (!dir) 358 - return false; 359 - 360 - struct dirent *entry; 361 - while ((entry = readdir(dir)) != NULL) { 362 - if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) 363 - continue; 364 - 365 - char full_path[PATH_MAX]; 366 - snprintf(full_path, sizeof(full_path), "%s/%s", current_path, 367 - entry->d_name); 368 - 369 - struct stat st; 370 - if (stat(full_path, &st) == -1) 371 - continue; 372 - 373 - if (S_ISDIR(st.st_mode)) { 374 - // Recurse into subdirectory 375 - pack_recursive(b, base_path, full_path, compress); 376 - } else if (S_ISREG(st.st_mode)) { 377 - // Calculate relative name (e.g., "subdir/file.txt") 378 - // We skip the base_path length + 1 (for the slash) 379 - const char *relative_name = full_path + strlen(base_path); 380 - if (*relative_name == '/') 381 - relative_name++; 382 - 383 - FILE *f = fopen(full_path, "rb"); 384 - if (!f) 385 - continue; 386 - 387 - uint8_t *buffer = malloc(st.st_size); 388 - if (fread(buffer, 1, st.st_size, f) == (size_t)st.st_size) { 389 - bindle_add(b, relative_name, buffer, st.st_size, 390 - compress ? BindleCompressZstd : BindleCompressNone); 391 - } 392 - fclose(f); 393 - free(buffer); 394 - } 395 - } 396 - closedir(dir); 397 - return true; 398 - } 399 - 400 - bool bindle_pack(Bindle *b, const char *src_dir, BindleCompress compress) { 401 - if (!b || !src_dir) 402 - return false; 403 - bool success = pack_recursive(b, src_dir, src_dir, compress); 404 - if (success) { 405 - return bindle_save(b); 406 - } 407 - return false; 408 - } 409 - 410 - /* --- Updated Unpack Implementation --- */ 411 - bool bindle_unpack(Bindle *b, const char *dest_dir) { 412 - if (!b || !dest_dir) 413 - return false; 414 - 415 - mkdir(dest_dir, 0755); 416 - 417 - for (uint64_t i = 0; i < b->count; i++) { 418 - size_t out_len = 0; 419 - uint8_t *data = bindle_read(b, b->entries[i].name, &out_len); 420 - 421 - if (data) { 422 - char full_path[PATH_MAX]; 423 - snprintf(full_path, sizeof(full_path), "%s/%s", dest_dir, 424 - b->entries[i].name); 425 - 426 - // Recreate directory structure before writing file 427 - ensure_dir_exists(full_path); 428 - 429 - FILE *out = fopen(full_path, "wb"); 430 - if (out) { 431 - fwrite(data, 1, out_len, out); 432 - fclose(out); 433 - } 434 - free(data); 435 - } 436 - } 437 - return true; 438 - }
-2
c/compile_flags.txt
··· 1 - -I../include 2 - -I/opt/homebrew/include
-75
c/test.py
··· 1 - import subprocess 2 - import os 3 - import hashlib 4 - import shutil 5 - 6 - def get_hash(data): 7 - return hashlib.sha256(data).hexdigest() 8 - 9 - def run_test(): 10 - test_file = "compat_test.bndl" 11 - secret_content = b"Consistency is the playground of the gods." 12 - with open("input.txt", "wb") as f: 13 - f.write(secret_content) 14 - 15 - print("--- Phase 1: Rust Create -> C Read ---") 16 - # 1. Create with Rust 17 - subprocess.run(["cargo", "run", "--", "add",test_file, "msg", "input.txt", "--compress"], check=True) 18 - 19 - # 2. Read with C (Assuming you compiled the C example to ./bindle_c) 20 - result_c = subprocess.run(["./bindle_c", "cat", test_file, "msg"], capture_output=True) 21 - 22 - if get_hash(result_c.stdout) == get_hash(secret_content): 23 - print("✅ SUCCESS: C successfully read Rust-compressed data.") 24 - else: 25 - print("❌ FAIL: C output does not match original content.") 26 - 27 - print("\n--- Phase 2: C Create -> Rust Read ---") 28 - # 3. Use C to add a different file (Assuming your C binary has an 'add' command) 29 - subprocess.run(["./bindle_c", "add", test_file, "c_msg", "input.txt", "1"], check=True) # 1 for compress 30 - 31 - # 4. Use Rust to list and verify 32 - result_rust = subprocess.run(["cargo", "run", "--", "cat", test_file, "c_msg"], capture_output=True) 33 - 34 - if get_hash(result_rust.stdout) == get_hash(secret_content): 35 - print("✅ SUCCESS: Rust successfully read C-compressed data.") 36 - else: 37 - print("❌ FAIL: Rust output does not match.") 38 - 39 - def run_pack_test(): 40 - test_file = "compat_test.bndl" 41 - src_dir = "test_dir_rust" 42 - dest_dir = "test_dir_c" 43 - 44 - # Cleanup 45 - if os.path.exists(src_dir): shutil.rmtree(src_dir) 46 - if os.path.exists(dest_dir): shutil.rmtree(dest_dir) 47 - os.makedirs(src_dir) 48 - 49 - # 1. Create files for Rust to pack 50 - with open(os.path.join(src_dir, "hello.txt"), "w") as f: 51 - f.write("Cross-language directory packing works!") 52 - 53 - print("--- Phase 3: Rust Pack -> C Unpack ---") 54 - 55 - # Rust Packs 56 - subprocess.run(["cargo", "run", "--", "pack", test_file, src_dir, "--compress"], check=True) 57 - 58 - # C Unpacks 59 - subprocess.run(["./bindle_c", "unpack", test_file, dest_dir], check=True) 60 - 61 - # Verify 62 - unpacked_file = os.path.join(dest_dir, "hello.txt") 63 - if os.path.exists(unpacked_file): 64 - with open(unpacked_file, "r") as f: 65 - content = f.read() 66 - if content == "Cross-language directory packing works!": 67 - print("✅ SUCCESS: C successfully unpacked Rust-packed directory.") 68 - else: 69 - print("❌ FAIL: Content mismatch in unpacked file.") 70 - else: 71 - print("❌ FAIL: Unpacked file missing.") 72 - 73 - if __name__ == "__main__": 74 - run_test() 75 - run_pack_test()
+1
cbindgen.toml
··· 12 12 "Footer" = "BindleFooter" 13 13 "Entry" = "BindleEntry" 14 14 "Compress" = "BindleCompress" 15 + "Stream" = "BindleStream"
+21
include/bindle.h
··· 20 20 21 21 typedef struct Bindle Bindle; 22 22 23 + typedef struct BindleStream BindleStream; 24 + 23 25 /** 24 26 * Open a bindle file from disk, the path paramter should be NUL terminated 25 27 */ ··· 34 36 const uint8_t *data, 35 37 size_t data_len, 36 38 enum BindleCompress compress); 39 + 40 + /** 41 + * Adds a new entry, the name should be NUL terminated, will the data can contain NUL characters since the length 42 + * is provided 43 + */ 44 + bool bindle_add_file(struct Bindle *ctx, 45 + const char *name, 46 + const char *path, 47 + enum BindleCompress compress); 37 48 38 49 /** 39 50 * Save any changed to disk ··· 82 93 bool bindle_unpack(struct Bindle *ctx, const char *dest_path); 83 94 84 95 bool bindle_pack(struct Bindle *ctx, const char *src_path, enum BindleCompress compress); 96 + 97 + bool bindle_exists(const struct Bindle *ctx, const char *name); 98 + 99 + struct BindleStream *bindle_stream_new(struct Bindle *ctx, 100 + const char *name, 101 + enum BindleCompress compress); 102 + 103 + bool bindle_stream_write(struct BindleStream *stream, const uint8_t *data, size_t len); 104 + 105 + bool bindle_stream_finish(struct BindleStream *stream); 85 106 86 107 #endif /* BINDLE_H */
+86 -1
src/ffi.rs
··· 1 1 use std::alloc::{Layout, dealloc}; 2 2 use std::ffi::CStr; 3 + use std::io::Write; 3 4 use std::mem; 4 5 use std::os::raw::c_char; 5 6 use std::slice; 6 7 7 - use crate::{Bindle, Compress}; 8 + use crate::{Bindle, Compress, Stream}; 8 9 9 10 /// Open a bindle file from disk, the path paramter should be NUL terminated 10 11 #[unsafe(no_mangle)] ··· 51 52 let b = &mut (*ctx); 52 53 53 54 b.add(name_str, data_slice, compress).is_ok() 55 + } 56 + } 57 + 58 + /// Adds a new entry, the name should be NUL terminated, will the data can contain NUL characters since the length 59 + /// is provided 60 + #[unsafe(no_mangle)] 61 + pub unsafe extern "C" fn bindle_add_file( 62 + ctx: *mut Bindle, 63 + name: *const c_char, 64 + path: *const c_char, 65 + compress: Compress, 66 + ) -> bool { 67 + if ctx.is_null() || name.is_null() || path.is_null() { 68 + return false; 69 + } 70 + 71 + unsafe { 72 + let name_str = match CStr::from_ptr(name).to_str() { 73 + Ok(s) => s, 74 + Err(_) => return false, 75 + }; 76 + 77 + let path_str = match CStr::from_ptr(path).to_str() { 78 + Ok(s) => s, 79 + Err(_) => return false, 80 + }; 81 + 82 + let b = &mut (*ctx); 83 + 84 + b.add_file(name_str, path_str, compress).is_ok() 54 85 } 55 86 } 56 87 ··· 256 287 let path = unsafe { CStr::from_ptr(src_path).to_string_lossy() }; 257 288 b.pack(path.as_ref(), compress).is_ok() 258 289 } 290 + 291 + #[unsafe(no_mangle)] 292 + pub unsafe extern "C" fn bindle_exists(ctx: *const Bindle, name: *const c_char) -> bool { 293 + if ctx.is_null() || name.is_null() { 294 + return false; 295 + } 296 + 297 + let b = unsafe { &*ctx }; 298 + let name_str = unsafe { 299 + match CStr::from_ptr(name).to_str() { 300 + Ok(s) => s, 301 + Err(_) => return false, 302 + } 303 + }; 304 + 305 + b.exists(name_str) 306 + } 307 + 308 + #[unsafe(no_mangle)] 309 + pub unsafe extern "C" fn bindle_stream_new<'a>( 310 + ctx: *mut Bindle, 311 + name: *const c_char, 312 + compress: Compress, 313 + ) -> *mut Stream<'a> { 314 + unsafe { 315 + let b = &mut *ctx; 316 + let name_str = CStr::from_ptr(name).to_string_lossy(); 317 + 318 + match b.stream(&name_str, compress) { 319 + Ok(stream) => Box::into_raw(Box::new(std::mem::transmute(stream))), 320 + Err(_) => std::ptr::null_mut(), 321 + } 322 + } 323 + } 324 + 325 + #[unsafe(no_mangle)] 326 + pub unsafe extern "C" fn bindle_stream_write( 327 + stream: *mut Stream, 328 + data: *const u8, 329 + len: usize, 330 + ) -> bool { 331 + unsafe { 332 + let s = &mut *stream; 333 + let chunk = std::slice::from_raw_parts(data, len); 334 + s.write_all(chunk).is_ok() 335 + } 336 + } 337 + 338 + #[unsafe(no_mangle)] 339 + pub unsafe extern "C" fn bindle_stream_finish(stream: *mut Stream) -> bool { 340 + // Reclaim memory from the Box and call finish() 341 + let s = unsafe { Box::from_raw(stream) }; 342 + s.finish().is_ok() 343 + }
+180 -1
src/lib.rs
··· 95 95 data_end: u64, 96 96 } 97 97 98 + pub enum Either<A, B> { 99 + Left(A), 100 + Right(B), 101 + } 102 + 103 + pub struct Stream<'a> { 104 + pub(crate) bindle: &'a mut Bindle, 105 + pub(crate) encoder: Option<zstd::Encoder<'a, std::fs::File>>, 106 + pub(crate) name: String, 107 + pub(crate) start_offset: u64, 108 + pub(crate) uncompressed_size: u64, 109 + } 110 + 111 + impl<'a> std::io::Write for Stream<'a> { 112 + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { 113 + self.write_chunk(buf)?; 114 + Ok(buf.len()) 115 + } 116 + 117 + fn flush(&mut self) -> io::Result<()> { 118 + Ok(()) 119 + } 120 + } 121 + 122 + impl<'a> Stream<'a> { 123 + pub fn write_chunk(&mut self, data: &[u8]) -> io::Result<()> { 124 + self.uncompressed_size += data.len() as u64; 125 + 126 + if let Some(encoder) = &mut self.encoder { 127 + encoder.write_all(data)?; 128 + } else { 129 + self.bindle.file.write_all(data)?; 130 + } 131 + 132 + Ok(()) 133 + } 134 + 135 + pub fn finish(self) -> io::Result<()> { 136 + let compression_type = if let Some(encoder) = self.encoder { 137 + encoder.finish()?; 138 + 1 139 + } else { 140 + 0 141 + }; 142 + 143 + let current_pos = self.bindle.file.stream_position()?; 144 + let compressed_size = current_pos - self.start_offset; 145 + 146 + // Handle 8-byte alignment padding 147 + let pad_len = pad::<8, u64>(current_pos); 148 + if pad_len > 0 { 149 + self.bindle.file.write_all(&vec![0u8; pad_len as usize])?; 150 + } 151 + 152 + self.bindle.data_end = current_pos + pad_len; 153 + 154 + let entry = Entry { 155 + offset: self.start_offset.to_le_bytes(), 156 + compressed_size: compressed_size.to_le_bytes(), 157 + uncompressed_size: self.uncompressed_size.to_le_bytes(), 158 + compression_type, 159 + name_len: (self.name.len() as u16).to_le_bytes(), 160 + ..Default::default() 161 + }; 162 + 163 + self.bindle.index.insert(self.name, entry); 164 + Ok(()) 165 + } 166 + } 167 + 98 168 impl Bindle { 99 169 /// Create a new bindle file, this will overwrite the existing file 100 170 pub fn create<P: AsRef<Path>>(path: P) -> io::Result<Self> { ··· 131 201 let mut file = opts.open(&path)?; 132 202 file.lock_shared()?; 133 203 let len = file.metadata()?.len(); 204 + 205 + // Handle completely new/empty files 134 206 if len == 0 { 135 207 file.write_all(BNDL_MAGIC)?; 136 208 return Ok(Self { ··· 142 214 }); 143 215 } 144 216 217 + // Safety check: File must be at least HEADER + FOOTER size (24 bytes) 218 + // This prevents "attempt to subtract with overflow" when calculating footer_pos 219 + if len < (HEADER_SIZE + FOOTER_SIZE) as u64 { 220 + return Err(io::Error::new( 221 + io::ErrorKind::InvalidData, 222 + "File too small to be a valid bindle", 223 + )); 224 + } 225 + 145 226 let mut header = [0u8; 8]; 146 227 file.read_exact(&mut header)?; 147 228 if &header != BNDL_MAGIC { ··· 149 230 } 150 231 151 232 let m = unsafe { Mmap::map(&file)? }; 233 + 234 + // Calculate footer position. Subtraction is now safe due to the check above. 152 235 let footer_pos = m.len() - FOOTER_SIZE; 153 236 let footer = Footer::read_from_bytes(&m[footer_pos..]).unwrap(); 154 237 ··· 158 241 159 242 let mut cursor = data_end as usize; 160 243 for _ in 0..count { 244 + // Ensure there is enough data left for an Entry header 245 + if cursor + ENTRY_SIZE > footer_pos { 246 + break; 247 + } 248 + 161 249 let entry = Entry::read_from_bytes(&m[cursor..cursor + ENTRY_SIZE]).unwrap(); 162 250 let n_start = cursor + ENTRY_SIZE; 251 + 252 + // Validate that the filename exists within the mapped bounds 253 + if n_start + entry.name_len() > footer_pos { 254 + break; 255 + } 256 + 163 257 let name = 164 258 String::from_utf8_lossy(&m[n_start..n_start + entry.name_len()]).into_owned(); 165 259 index.insert(name, entry); 260 + 166 261 let total = ENTRY_SIZE + entry.name_len(); 167 262 cursor += (total + (BNDL_ALIGN - 1)) & !(BNDL_ALIGN - 1); 168 263 } ··· 176 271 }) 177 272 } 178 273 274 + fn should_auto_compress(&self, compress: Compress, len: usize) -> bool { 275 + compress == Compress::Zstd || (compress == Compress::Auto && len > AUTO_COMPRESS_THRESHOLD) 276 + } 277 + 179 278 pub fn add(&mut self, name: &str, data: &[u8], compress: Compress) -> io::Result<()> { 180 - let compress = compress == Compress::Zstd || data.len() > AUTO_COMPRESS_THRESHOLD; 279 + let compress = self.should_auto_compress(compress, data.len()); 181 280 let (processed, c_type) = if compress { 182 281 (Cow::Owned(zstd::encode_all(data, 3)?), Compress::Zstd) 183 282 } else { ··· 206 305 }; 207 306 208 307 self.index.insert(name.to_string(), entry); 308 + Ok(()) 309 + } 310 + 311 + pub fn add_file( 312 + &mut self, 313 + name: &str, 314 + path: impl AsRef<Path>, 315 + compress: Compress, 316 + ) -> io::Result<()> { 317 + let mut stream = self.stream(name, compress)?; 318 + let mut src = std::fs::File::open(path)?; 319 + std::io::copy(&mut src, &mut stream)?; 209 320 Ok(()) 210 321 } 211 322 ··· 326 437 } 327 438 } 328 439 440 + /// The number of entries 329 441 pub fn len(&self) -> usize { 330 442 self.index.len() 331 443 } 332 444 445 + /// Returns true if there are no entries 333 446 pub fn is_empty(&self) -> bool { 334 447 self.index.is_empty() 335 448 } 336 449 450 + /// Direct readonly access to the index 337 451 pub fn index(&self) -> &BTreeMap<String, Entry> { 338 452 &self.index 339 453 } 340 454 455 + /// Clear all entries 456 + pub fn clear(&mut self) { 457 + self.index.clear() 458 + } 459 + 460 + /// Checks if an entry exists in the archive index. 461 + pub fn exists(&self, name: &str) -> bool { 462 + self.index.contains_key(name) 463 + } 464 + 341 465 /// Recursively packs a directory into the archive. 342 466 pub fn pack<P: AsRef<Path>>(&mut self, src_dir: P, compress: Compress) -> io::Result<()> { 343 467 self.pack_recursive(src_dir.as_ref(), src_dir.as_ref(), compress) ··· 368 492 /// Unpacks all archive entries to a destination directory. 369 493 pub fn unpack<P: AsRef<Path>>(&self, dest: P) -> io::Result<()> { 370 494 let dest_path = dest.as_ref(); 495 + if let Some(parent) = dest_path.parent() { 496 + std::fs::create_dir_all(parent)?; 497 + } 371 498 for (name, _) in &self.index { 372 499 if let Some(data) = self.read(name) { 373 500 let file_path = dest_path.join(name); ··· 379 506 } 380 507 Ok(()) 381 508 } 509 + 510 + pub fn stream<'a>(&'a mut self, name: &str, compress: Compress) -> io::Result<Stream<'a>> { 511 + self.file.seek(SeekFrom::Start(self.data_end))?; 512 + let compress = self.should_auto_compress(compress, 0); 513 + let f = self.file.try_clone()?; 514 + let start_offset = self.data_end; 515 + Ok(Stream { 516 + name: name.to_string(), 517 + bindle: self, 518 + encoder: if compress { 519 + Some(zstd::Encoder::new(f, 3)?) 520 + } else { 521 + None 522 + }, 523 + start_offset, 524 + uncompressed_size: 0, 525 + }) 526 + } 382 527 } 383 528 384 529 impl Drop for Bindle { ··· 588 733 fs::remove_dir_all(src_dir).ok(); 589 734 fs::remove_dir_all(out_dir).ok(); 590 735 fs::remove_file(bindle_path).ok(); 736 + } 737 + 738 + #[test] 739 + fn test_streaming_manual_chunks() { 740 + let path = "test_stream.bindl"; 741 + let _ = std::fs::remove_file(path); 742 + let chunk1 = b"Hello "; 743 + let chunk2 = b"Streaming "; 744 + let chunk3 = b"World!"; 745 + let expected = b"Hello Streaming World!"; 746 + 747 + { 748 + let mut b = Bindle::open(path).expect("Failed to open"); 749 + // Start a stream without compression 750 + let mut s = b 751 + .stream("streamed_file.txt", Compress::None) 752 + .expect("Failed to start stream"); 753 + 754 + // Write chunks manually 755 + s.write_chunk(chunk1).unwrap(); 756 + s.write_chunk(chunk2).unwrap(); 757 + s.write_chunk(chunk3).unwrap(); 758 + 759 + s.finish().expect("Failed to finish stream"); 760 + b.save().expect("Failed to save"); 761 + } 762 + 763 + // Verification 764 + let b = Bindle::open(path).expect("Failed to reopen"); 765 + let result = b.read("streamed_file.txt").expect("Entry not found"); 766 + assert_eq!(result.as_ref(), expected); 767 + assert_eq!(result.len(), expected.len()); 768 + 769 + let _ = std::fs::remove_file(path); 591 770 } 592 771 }