an efficient binary archive format
remove c code
+301
-624
+12
-1
Makefile
+12
-1
Makefile
···
1
+
lib:
2
+
cargo build --release
3
+
cp target/release/libbindle_file.a .
4
+
cp target/release/libbindle_file.so libbindle.so || cp target/release/libbindle_file.dylib libbindle.dylib
5
+
6
+
install:
7
+
cp include/bindle.h /usr/local/include/bindle.h
8
+
cp libbbindle.* /usr/local/lib
9
+
10
+
uninstall:
11
+
rm -f /usr/local/include/bindle.h /usr/local/lib/libbindle.*
12
+
1
13
test:
2
14
cargo test
3
-
make -C c test
+1
-4
README.md
+1
-4
README.md
···
1
1
# bindle-file
2
2
3
-
`bindle` is a general purpose, multi-file archive format with support for zstd and direct reads for uncompressed values.
3
+
`bindle` is a general purpose, binary archive format designed for efficient reads and writes.
4
4
5
-
This repository contains `bindle-file` for Rust, which can also be used to build C libraries in
6
-
`target/release/libbindle_file.a` and `target/release/libbindle_file.so`. `c/bindle.c` can also
7
-
be used as a drop-in replacement for the Rust implementation in C projects.
8
5
-8
c/Makefile
-8
c/Makefile
···
1
-
.PHONY: test
2
-
test: check
3
-
python3 test.py a.bindl
4
-
clang -Wall -Wextra -Wpedantic -I ../include -o bindle_c bindle-cli.c bindle.c `pkg-config --cflags --libs libzstd`
5
-
6
-
check:
7
-
clang -fsanitize=address -Wall -Wextra -Wpedantic -I ../include -o bindle_c bindle-cli.c bindle.c `pkg-config --cflags --libs libzstd`
8
-
-94
c/bindle-cli.c
-94
c/bindle-cli.c
···
1
-
#include "bindle.h"
2
-
#include <stdio.h>
3
-
#include <stdlib.h>
4
-
#include <string.h>
5
-
6
-
void print_usage(const char *prog) {
7
-
printf("Usage: %s <command> <bindle_file> [args]\n", prog);
8
-
printf("Commands:\n");
9
-
printf(" list List all entries\n");
10
-
printf(" cat <name> Output entry content to stdout\n");
11
-
printf(" add <name> <file> Add a single file to the archive\n");
12
-
printf(" pack <src_dir> Pack a directory into the archive\n");
13
-
printf(" unpack <dest_dir> Unpack archive to a directory\n");
14
-
printf(" vacuum Reclaim space from shadowed entries\n");
15
-
}
16
-
17
-
int main(int argc, char **argv) {
18
-
if (argc < 3) {
19
-
print_usage(argv[0]);
20
-
return 1;
21
-
}
22
-
23
-
// Switched back: command first, then bindle_file
24
-
const char *cmd = argv[1];
25
-
const char *db_path = argv[2];
26
-
27
-
Bindle *b = bindle_open(db_path);
28
-
if (!b) {
29
-
fprintf(stderr, "Error: Could not open bindle '%s'\n", db_path);
30
-
return 1;
31
-
}
32
-
33
-
if (strcmp(cmd, "list") == 0) {
34
-
uint64_t count = bindle_length(b);
35
-
printf("%-30s\n", "NAME");
36
-
printf("------------------------------\n");
37
-
for (uint64_t i = 0; i < count; i++) {
38
-
size_t namelen = 0;
39
-
const char *name = bindle_entry_name(b, i, &namelen);
40
-
printf("%-30s\n", name);
41
-
}
42
-
} else if (strcmp(cmd, "cat") == 0) {
43
-
if (argc < 4) {
44
-
fprintf(stderr, "Usage: %s cat <file> <name>\n", argv[0]);
45
-
bindle_close(b);
46
-
return 1;
47
-
}
48
-
size_t out_len = 0;
49
-
uint8_t *data = bindle_read(b, argv[3], &out_len);
50
-
if (data) {
51
-
fwrite(data, 1, out_len, stdout);
52
-
free(data);
53
-
}
54
-
} else if (strcmp(cmd, "add") == 0) {
55
-
if (argc < 5) {
56
-
fprintf(stderr, "Usage: %s add <file> <name> <src_path>\n", argv[0]);
57
-
bindle_close(b);
58
-
return 1;
59
-
}
60
-
FILE *inf = fopen(argv[4], "rb");
61
-
if (inf) {
62
-
fseek(inf, 0, SEEK_END);
63
-
size_t len = ftell(inf);
64
-
fseek(inf, 0, SEEK_SET);
65
-
uint8_t *buf = malloc(len);
66
-
fread(buf, 1, len, inf);
67
-
fclose(inf);
68
-
69
-
if (bindle_add(b, argv[3], buf, len, true)) {
70
-
bindle_save(b);
71
-
}
72
-
free(buf);
73
-
}
74
-
} else if (strcmp(cmd, "pack") == 0) {
75
-
if (argc < 4) {
76
-
fprintf(stderr, "Usage: %s pack <file> <src_dir>\n", argv[0]);
77
-
bindle_close(b);
78
-
return 1;
79
-
}
80
-
bindle_pack(b, argv[3], true);
81
-
} else if (strcmp(cmd, "unpack") == 0) {
82
-
if (argc < 4) {
83
-
fprintf(stderr, "Usage: %s unpack <file> <dest_dir>\n", argv[0]);
84
-
bindle_close(b);
85
-
return 1;
86
-
}
87
-
bindle_unpack(b, argv[3]);
88
-
} else if (strcmp(cmd, "vacuum") == 0) {
89
-
bindle_vacuum(b);
90
-
}
91
-
92
-
bindle_close(b);
93
-
return 0;
94
-
}
-438
c/bindle.c
-438
c/bindle.c
···
1
-
#include "bindle.h"
2
-
3
-
#include <dirent.h>
4
-
#include <errno.h>
5
-
#include <limits.h>
6
-
#include <stdio.h>
7
-
#include <stdlib.h>
8
-
#include <string.h>
9
-
#include <sys/file.h>
10
-
#include <sys/stat.h>
11
-
#include <zstd.h>
12
-
13
-
#define BNDL_MAGIC "BINDL001"
14
-
#define BNDL_ALIGN 8
15
-
#define ALIGN_UP(n, m) (((n) + (m) - 1) & ~((m) - 1))
16
-
17
-
/* --- Private Disk Structures --- */
18
-
#pragma pack(push, 1)
19
-
typedef struct {
20
-
uint64_t offset;
21
-
uint64_t compressed_size;
22
-
uint64_t uncompressed_size;
23
-
uint32_t crc32;
24
-
uint16_t name_len;
25
-
uint8_t compression_type;
26
-
uint8_t _reserved;
27
-
} BindleEntryRaw;
28
-
29
-
typedef struct {
30
-
uint64_t index_offset;
31
-
uint64_t entry_count;
32
-
} BindleFooterRaw;
33
-
#pragma pack(pop)
34
-
35
-
/* --- Private In-Memory Structures --- */
36
-
typedef struct {
37
-
BindleEntryRaw meta;
38
-
char *name;
39
-
} BindleEntry;
40
-
41
-
struct Bindle {
42
-
char *path;
43
-
FILE *fp;
44
-
BindleEntry *entries;
45
-
uint64_t count;
46
-
uint64_t data_end;
47
-
};
48
-
49
-
/* --- API Implementation --- */
50
-
51
-
Bindle *bindle_open(const char *path) {
52
-
FILE *fp = fopen(path, "r+b");
53
-
if (!fp) {
54
-
fp = fopen(path, "w+b");
55
-
}
56
-
if (!fp)
57
-
return NULL;
58
-
59
-
flock(fileno(fp), LOCK_SH);
60
-
61
-
Bindle *b = calloc(1, sizeof(Bindle));
62
-
b->path = strdup(path);
63
-
b->fp = fp;
64
-
65
-
fseek(fp, 0, SEEK_END);
66
-
long file_size = ftell(fp);
67
-
68
-
if (file_size == 0) {
69
-
fwrite(BNDL_MAGIC, 8, 1, fp);
70
-
b->data_end = 8;
71
-
return b;
72
-
}
73
-
74
-
// Header check
75
-
char magic[8];
76
-
fseek(fp, 0, SEEK_SET);
77
-
if (fread(magic, 8, 1, fp) != 1 || memcmp(magic, BNDL_MAGIC, 8) != 0) {
78
-
bindle_close(b);
79
-
return NULL;
80
-
}
81
-
82
-
// Footer parse
83
-
BindleFooterRaw footer;
84
-
fseek(fp, file_size - sizeof(BindleFooterRaw), SEEK_SET);
85
-
if (fread(&footer, sizeof(BindleFooterRaw), 1, fp) != 1) {
86
-
bindle_close(b);
87
-
return NULL;
88
-
}
89
-
90
-
b->count = footer.entry_count;
91
-
b->data_end = footer.index_offset;
92
-
b->entries = malloc(sizeof(BindleEntry) * b->count);
93
-
94
-
// Index parse
95
-
fseek(fp, footer.index_offset, SEEK_SET);
96
-
for (uint64_t i = 0; i < b->count; i++) {
97
-
fread(&b->entries[i].meta, sizeof(BindleEntryRaw), 1, fp);
98
-
b->entries[i].name = malloc(b->entries[i].meta.name_len + 1);
99
-
fread(b->entries[i].name, b->entries[i].meta.name_len, 1, fp);
100
-
b->entries[i].name[b->entries[i].meta.name_len] = '\0';
101
-
102
-
size_t consumed = sizeof(BindleEntryRaw) + b->entries[i].meta.name_len;
103
-
fseek(fp, ALIGN_UP(consumed, BNDL_ALIGN) - consumed, SEEK_CUR);
104
-
}
105
-
return b;
106
-
}
107
-
108
-
bool bindle_add(Bindle *b, const char *name, const uint8_t *data, size_t len,
109
-
BindleCompress compress) {
110
-
if (!b || !name)
111
-
return false;
112
-
113
-
size_t c_size = len;
114
-
void *write_ptr = (void *)data;
115
-
void *comp_buf = NULL;
116
-
117
-
if (compress == BindleCompressZstd) {
118
-
size_t bound = ZSTD_compressBound(len);
119
-
comp_buf = malloc(bound);
120
-
c_size = ZSTD_compress(comp_buf, bound, data, len, 3);
121
-
if (ZSTD_isError(c_size)) {
122
-
free(comp_buf);
123
-
return false;
124
-
}
125
-
write_ptr = comp_buf;
126
-
}
127
-
128
-
// 1. Write data at current data_end
129
-
fseek(b->fp, b->data_end, SEEK_SET);
130
-
uint64_t offset = ftell(b->fp);
131
-
fwrite(write_ptr, 1, c_size, b->fp);
132
-
133
-
// 2. Align data_end
134
-
size_t pad = ALIGN_UP(c_size, BNDL_ALIGN) - c_size;
135
-
if (pad > 0) {
136
-
uint8_t zero[8] = {0};
137
-
fwrite(zero, 1, pad, b->fp);
138
-
}
139
-
b->data_end = ftell(b->fp);
140
-
141
-
// 3. Shadowing: Check if name already exists
142
-
for (uint64_t i = 0; i < b->count; i++) {
143
-
if (strcmp(b->entries[i].name, name) == 0) {
144
-
b->entries[i].meta.offset = offset;
145
-
b->entries[i].meta.compressed_size = c_size;
146
-
b->entries[i].meta.uncompressed_size = len;
147
-
b->entries[i].meta.compression_type = compress ? 1 : 0;
148
-
if (comp_buf)
149
-
free(comp_buf);
150
-
return true;
151
-
}
152
-
}
153
-
154
-
// 4. New Entry
155
-
b->entries = realloc(b->entries, sizeof(BindleEntry) * (b->count + 1));
156
-
BindleEntry *e = &b->entries[b->count++];
157
-
e->name = strdup(name);
158
-
e->meta = (BindleEntryRaw){offset, c_size, len, 0, (uint16_t)strlen(name),
159
-
compress, 0};
160
-
161
-
if (comp_buf)
162
-
free(comp_buf);
163
-
return true;
164
-
}
165
-
166
-
uint8_t *bindle_read(Bindle *b, const char *name, size_t *out_len) {
167
-
for (uint64_t i = 0; i < b->count; i++) {
168
-
if (strcmp(b->entries[i].name, name) == 0) {
169
-
BindleEntryRaw *m = &b->entries[i].meta;
170
-
uint8_t *c_buf = malloc(m->compressed_size);
171
-
fseek(b->fp, m->offset, SEEK_SET);
172
-
fread(c_buf, 1, m->compressed_size, b->fp);
173
-
174
-
if (m->compression_type == BindleCompressZstd) {
175
-
uint8_t *u_buf = malloc(m->uncompressed_size);
176
-
size_t actual = ZSTD_decompress(u_buf, m->uncompressed_size, c_buf,
177
-
m->compressed_size);
178
-
free(c_buf);
179
-
*out_len = actual;
180
-
return u_buf;
181
-
}
182
-
*out_len = m->compressed_size;
183
-
return c_buf;
184
-
}
185
-
}
186
-
return NULL;
187
-
}
188
-
189
-
const uint8_t *bindle_read_uncompressed_direct(Bindle *b, const char *name,
190
-
size_t *out_len) {
191
-
for (uint64_t i = 0; i < b->count; i++) {
192
-
if (strcmp(b->entries[i].name, name) == 0) {
193
-
BindleEntryRaw *m = &b->entries[i].meta;
194
-
if (m->compression_type != BindleCompressNone)
195
-
return NULL;
196
-
197
-
uint8_t *buf = malloc(m->uncompressed_size);
198
-
fseek(b->fp, m->offset, SEEK_SET);
199
-
fread(buf, 1, m->uncompressed_size, b->fp);
200
-
*out_len = m->uncompressed_size;
201
-
return buf;
202
-
}
203
-
}
204
-
return NULL;
205
-
}
206
-
207
-
bool bindle_save(Bindle *b) {
208
-
flock(fileno(b->fp), LOCK_EX);
209
-
fseek(b->fp, b->data_end, SEEK_SET);
210
-
uint64_t index_start = b->data_end;
211
-
212
-
for (uint64_t i = 0; i < b->count; i++) {
213
-
fwrite(&b->entries[i].meta, sizeof(BindleEntryRaw), 1, b->fp);
214
-
fwrite(b->entries[i].name, 1, b->entries[i].meta.name_len, b->fp);
215
-
size_t consumed = sizeof(BindleEntryRaw) + b->entries[i].meta.name_len;
216
-
size_t pad = ALIGN_UP(consumed, BNDL_ALIGN) - consumed;
217
-
if (pad > 0) {
218
-
uint8_t zero[8] = {0};
219
-
fwrite(zero, 1, pad, b->fp);
220
-
}
221
-
}
222
-
223
-
BindleFooterRaw footer = {index_start, b->count};
224
-
fwrite(&footer, sizeof(BindleFooterRaw), 1, b->fp);
225
-
fflush(b->fp);
226
-
flock(fileno(b->fp), LOCK_SH);
227
-
return true;
228
-
}
229
-
230
-
size_t bindle_length(const Bindle *b) { return b ? b->count : 0; }
231
-
232
-
const char *bindle_entry_name(const Bindle *b, size_t index, size_t *namelen) {
233
-
if (!b || index >= b->count)
234
-
return NULL;
235
-
*namelen = b->entries[index].meta.name_len;
236
-
return b->entries[index].name;
237
-
}
238
-
239
-
void bindle_free_buffer(uint8_t *ptr) { free(ptr); }
240
-
241
-
void bindle_close(Bindle *b) {
242
-
if (!b)
243
-
return;
244
-
flock(fileno(b->fp), LOCK_UN);
245
-
for (uint64_t i = 0; i < b->count; i++)
246
-
free(b->entries[i].name);
247
-
free(b->entries);
248
-
fclose(b->fp);
249
-
free(b->path);
250
-
free(b);
251
-
}
252
-
253
-
bool bindle_vacuum(Bindle *b) {
254
-
if (!b)
255
-
return false;
256
-
257
-
char tmp_path[1024];
258
-
snprintf(tmp_path, sizeof(tmp_path), "%s.tmp", b->path);
259
-
FILE *out = fopen(tmp_path, "wb");
260
-
if (!out)
261
-
return false;
262
-
263
-
// 1. Write Header
264
-
fwrite(BNDL_MAGIC, 8, 1, out);
265
-
uint64_t current_offset = 8;
266
-
267
-
// 2. Copy Live Data to Temp File
268
-
for (uint64_t i = 0; i < b->count; i++) {
269
-
uint64_t size = b->entries[i].meta.compressed_size;
270
-
uint8_t *buf = malloc(size);
271
-
272
-
fseek(b->fp, b->entries[i].meta.offset, SEEK_SET);
273
-
fread(buf, 1, size, b->fp);
274
-
275
-
fseek(out, current_offset, SEEK_SET);
276
-
fwrite(buf, 1, size, out);
277
-
278
-
// Update the in-memory metadata with the new offset
279
-
b->entries[i].meta.offset = current_offset;
280
-
281
-
size_t pad = ALIGN_UP(size, BNDL_ALIGN) - size;
282
-
if (pad > 0) {
283
-
uint8_t zero[8] = {0};
284
-
fwrite(zero, 1, pad, out);
285
-
}
286
-
current_offset += size + pad;
287
-
free(buf);
288
-
}
289
-
290
-
// 3. Write Index and Footer to the Temp File (Matching Rust .save() logic)
291
-
uint64_t index_start = current_offset;
292
-
for (uint64_t i = 0; i < b->count; i++) {
293
-
fwrite(&b->entries[i].meta, sizeof(BindleEntryRaw), 1, out);
294
-
fwrite(b->entries[i].name, 1, b->entries[i].meta.name_len, out);
295
-
296
-
size_t consumed = sizeof(BindleEntryRaw) + b->entries[i].meta.name_len;
297
-
size_t pad = ALIGN_UP(consumed, BNDL_ALIGN) - consumed;
298
-
if (pad > 0) {
299
-
uint8_t zero[8] = {0};
300
-
fwrite(zero, 1, pad, out);
301
-
}
302
-
}
303
-
304
-
BindleFooterRaw footer = {index_start, b->count};
305
-
fwrite(&footer, sizeof(BindleFooterRaw), 1, out);
306
-
307
-
// 4. CRITICAL: Close and Unlock handles before Rename
308
-
fflush(out);
309
-
fclose(out); // Close the temp file handle
310
-
311
-
flock(fileno(b->fp), LOCK_UN); // Explicitly unlock the original file
312
-
fclose(b->fp); // Close the original file handle
313
-
b->fp = NULL;
314
-
315
-
// 5. Atomic Rename
316
-
if (rename(tmp_path, b->path) != 0) {
317
-
// If rename fails, we are in a bad state; attempt to re-open original
318
-
b->fp = fopen(b->path, "r+b");
319
-
return false;
320
-
}
321
-
322
-
// 6. Re-open the new primary file
323
-
b->fp = fopen(b->path, "r+b");
324
-
if (!b->fp)
325
-
return false;
326
-
327
-
flock(fileno(b->fp), LOCK_SH);
328
-
b->data_end = index_start;
329
-
330
-
return true;
331
-
}
332
-
333
-
/* --- Helper: Recursive directory creation (mkdir -p) --- */
334
-
static void ensure_dir_exists(const char *path) {
335
-
char tmp[1024];
336
-
char *p = NULL;
337
-
size_t len;
338
-
339
-
snprintf(tmp, sizeof(tmp), "%s", path);
340
-
len = strlen(tmp);
341
-
if (tmp[len - 1] == '/')
342
-
tmp[len - 1] = 0;
343
-
344
-
for (p = tmp + 1; *p; p++) {
345
-
if (*p == '/') {
346
-
*p = 0;
347
-
mkdir(tmp, 0755);
348
-
*p = '/';
349
-
}
350
-
}
351
-
}
352
-
353
-
/* --- Recursive Pack Implementation --- */
354
-
static bool pack_recursive(Bindle *b, const char *base_path,
355
-
const char *current_path, bool compress) {
356
-
DIR *dir = opendir(current_path);
357
-
if (!dir)
358
-
return false;
359
-
360
-
struct dirent *entry;
361
-
while ((entry = readdir(dir)) != NULL) {
362
-
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
363
-
continue;
364
-
365
-
char full_path[PATH_MAX];
366
-
snprintf(full_path, sizeof(full_path), "%s/%s", current_path,
367
-
entry->d_name);
368
-
369
-
struct stat st;
370
-
if (stat(full_path, &st) == -1)
371
-
continue;
372
-
373
-
if (S_ISDIR(st.st_mode)) {
374
-
// Recurse into subdirectory
375
-
pack_recursive(b, base_path, full_path, compress);
376
-
} else if (S_ISREG(st.st_mode)) {
377
-
// Calculate relative name (e.g., "subdir/file.txt")
378
-
// We skip the base_path length + 1 (for the slash)
379
-
const char *relative_name = full_path + strlen(base_path);
380
-
if (*relative_name == '/')
381
-
relative_name++;
382
-
383
-
FILE *f = fopen(full_path, "rb");
384
-
if (!f)
385
-
continue;
386
-
387
-
uint8_t *buffer = malloc(st.st_size);
388
-
if (fread(buffer, 1, st.st_size, f) == (size_t)st.st_size) {
389
-
bindle_add(b, relative_name, buffer, st.st_size,
390
-
compress ? BindleCompressZstd : BindleCompressNone);
391
-
}
392
-
fclose(f);
393
-
free(buffer);
394
-
}
395
-
}
396
-
closedir(dir);
397
-
return true;
398
-
}
399
-
400
-
bool bindle_pack(Bindle *b, const char *src_dir, BindleCompress compress) {
401
-
if (!b || !src_dir)
402
-
return false;
403
-
bool success = pack_recursive(b, src_dir, src_dir, compress);
404
-
if (success) {
405
-
return bindle_save(b);
406
-
}
407
-
return false;
408
-
}
409
-
410
-
/* --- Updated Unpack Implementation --- */
411
-
bool bindle_unpack(Bindle *b, const char *dest_dir) {
412
-
if (!b || !dest_dir)
413
-
return false;
414
-
415
-
mkdir(dest_dir, 0755);
416
-
417
-
for (uint64_t i = 0; i < b->count; i++) {
418
-
size_t out_len = 0;
419
-
uint8_t *data = bindle_read(b, b->entries[i].name, &out_len);
420
-
421
-
if (data) {
422
-
char full_path[PATH_MAX];
423
-
snprintf(full_path, sizeof(full_path), "%s/%s", dest_dir,
424
-
b->entries[i].name);
425
-
426
-
// Recreate directory structure before writing file
427
-
ensure_dir_exists(full_path);
428
-
429
-
FILE *out = fopen(full_path, "wb");
430
-
if (out) {
431
-
fwrite(data, 1, out_len, out);
432
-
fclose(out);
433
-
}
434
-
free(data);
435
-
}
436
-
}
437
-
return true;
438
-
}
-75
c/test.py
-75
c/test.py
···
1
-
import subprocess
2
-
import os
3
-
import hashlib
4
-
import shutil
5
-
6
-
def get_hash(data):
7
-
return hashlib.sha256(data).hexdigest()
8
-
9
-
def run_test():
10
-
test_file = "compat_test.bndl"
11
-
secret_content = b"Consistency is the playground of the gods."
12
-
with open("input.txt", "wb") as f:
13
-
f.write(secret_content)
14
-
15
-
print("--- Phase 1: Rust Create -> C Read ---")
16
-
# 1. Create with Rust
17
-
subprocess.run(["cargo", "run", "--", "add",test_file, "msg", "input.txt", "--compress"], check=True)
18
-
19
-
# 2. Read with C (Assuming you compiled the C example to ./bindle_c)
20
-
result_c = subprocess.run(["./bindle_c", "cat", test_file, "msg"], capture_output=True)
21
-
22
-
if get_hash(result_c.stdout) == get_hash(secret_content):
23
-
print("✅ SUCCESS: C successfully read Rust-compressed data.")
24
-
else:
25
-
print("❌ FAIL: C output does not match original content.")
26
-
27
-
print("\n--- Phase 2: C Create -> Rust Read ---")
28
-
# 3. Use C to add a different file (Assuming your C binary has an 'add' command)
29
-
subprocess.run(["./bindle_c", "add", test_file, "c_msg", "input.txt", "1"], check=True) # 1 for compress
30
-
31
-
# 4. Use Rust to list and verify
32
-
result_rust = subprocess.run(["cargo", "run", "--", "cat", test_file, "c_msg"], capture_output=True)
33
-
34
-
if get_hash(result_rust.stdout) == get_hash(secret_content):
35
-
print("✅ SUCCESS: Rust successfully read C-compressed data.")
36
-
else:
37
-
print("❌ FAIL: Rust output does not match.")
38
-
39
-
def run_pack_test():
40
-
test_file = "compat_test.bndl"
41
-
src_dir = "test_dir_rust"
42
-
dest_dir = "test_dir_c"
43
-
44
-
# Cleanup
45
-
if os.path.exists(src_dir): shutil.rmtree(src_dir)
46
-
if os.path.exists(dest_dir): shutil.rmtree(dest_dir)
47
-
os.makedirs(src_dir)
48
-
49
-
# 1. Create files for Rust to pack
50
-
with open(os.path.join(src_dir, "hello.txt"), "w") as f:
51
-
f.write("Cross-language directory packing works!")
52
-
53
-
print("--- Phase 3: Rust Pack -> C Unpack ---")
54
-
55
-
# Rust Packs
56
-
subprocess.run(["cargo", "run", "--", "pack", test_file, src_dir, "--compress"], check=True)
57
-
58
-
# C Unpacks
59
-
subprocess.run(["./bindle_c", "unpack", test_file, dest_dir], check=True)
60
-
61
-
# Verify
62
-
unpacked_file = os.path.join(dest_dir, "hello.txt")
63
-
if os.path.exists(unpacked_file):
64
-
with open(unpacked_file, "r") as f:
65
-
content = f.read()
66
-
if content == "Cross-language directory packing works!":
67
-
print("✅ SUCCESS: C successfully unpacked Rust-packed directory.")
68
-
else:
69
-
print("❌ FAIL: Content mismatch in unpacked file.")
70
-
else:
71
-
print("❌ FAIL: Unpacked file missing.")
72
-
73
-
if __name__ == "__main__":
74
-
run_test()
75
-
run_pack_test()
+1
cbindgen.toml
+1
cbindgen.toml
+21
include/bindle.h
+21
include/bindle.h
···
20
20
21
21
typedef struct Bindle Bindle;
22
22
23
+
typedef struct BindleStream BindleStream;
24
+
23
25
/**
24
26
* Open a bindle file from disk, the path paramter should be NUL terminated
25
27
*/
···
34
36
const uint8_t *data,
35
37
size_t data_len,
36
38
enum BindleCompress compress);
39
+
40
+
/**
41
+
* Adds a new entry, the name should be NUL terminated, will the data can contain NUL characters since the length
42
+
* is provided
43
+
*/
44
+
bool bindle_add_file(struct Bindle *ctx,
45
+
const char *name,
46
+
const char *path,
47
+
enum BindleCompress compress);
37
48
38
49
/**
39
50
* Save any changed to disk
···
82
93
bool bindle_unpack(struct Bindle *ctx, const char *dest_path);
83
94
84
95
bool bindle_pack(struct Bindle *ctx, const char *src_path, enum BindleCompress compress);
96
+
97
+
bool bindle_exists(const struct Bindle *ctx, const char *name);
98
+
99
+
struct BindleStream *bindle_stream_new(struct Bindle *ctx,
100
+
const char *name,
101
+
enum BindleCompress compress);
102
+
103
+
bool bindle_stream_write(struct BindleStream *stream, const uint8_t *data, size_t len);
104
+
105
+
bool bindle_stream_finish(struct BindleStream *stream);
85
106
86
107
#endif /* BINDLE_H */
+86
-1
src/ffi.rs
+86
-1
src/ffi.rs
···
1
1
use std::alloc::{Layout, dealloc};
2
2
use std::ffi::CStr;
3
+
use std::io::Write;
3
4
use std::mem;
4
5
use std::os::raw::c_char;
5
6
use std::slice;
6
7
7
-
use crate::{Bindle, Compress};
8
+
use crate::{Bindle, Compress, Stream};
8
9
9
10
/// Open a bindle file from disk, the path paramter should be NUL terminated
10
11
#[unsafe(no_mangle)]
···
51
52
let b = &mut (*ctx);
52
53
53
54
b.add(name_str, data_slice, compress).is_ok()
55
+
}
56
+
}
57
+
58
+
/// Adds a new entry, the name should be NUL terminated, will the data can contain NUL characters since the length
59
+
/// is provided
60
+
#[unsafe(no_mangle)]
61
+
pub unsafe extern "C" fn bindle_add_file(
62
+
ctx: *mut Bindle,
63
+
name: *const c_char,
64
+
path: *const c_char,
65
+
compress: Compress,
66
+
) -> bool {
67
+
if ctx.is_null() || name.is_null() || path.is_null() {
68
+
return false;
69
+
}
70
+
71
+
unsafe {
72
+
let name_str = match CStr::from_ptr(name).to_str() {
73
+
Ok(s) => s,
74
+
Err(_) => return false,
75
+
};
76
+
77
+
let path_str = match CStr::from_ptr(path).to_str() {
78
+
Ok(s) => s,
79
+
Err(_) => return false,
80
+
};
81
+
82
+
let b = &mut (*ctx);
83
+
84
+
b.add_file(name_str, path_str, compress).is_ok()
54
85
}
55
86
}
56
87
···
256
287
let path = unsafe { CStr::from_ptr(src_path).to_string_lossy() };
257
288
b.pack(path.as_ref(), compress).is_ok()
258
289
}
290
+
291
+
#[unsafe(no_mangle)]
292
+
pub unsafe extern "C" fn bindle_exists(ctx: *const Bindle, name: *const c_char) -> bool {
293
+
if ctx.is_null() || name.is_null() {
294
+
return false;
295
+
}
296
+
297
+
let b = unsafe { &*ctx };
298
+
let name_str = unsafe {
299
+
match CStr::from_ptr(name).to_str() {
300
+
Ok(s) => s,
301
+
Err(_) => return false,
302
+
}
303
+
};
304
+
305
+
b.exists(name_str)
306
+
}
307
+
308
+
#[unsafe(no_mangle)]
309
+
pub unsafe extern "C" fn bindle_stream_new<'a>(
310
+
ctx: *mut Bindle,
311
+
name: *const c_char,
312
+
compress: Compress,
313
+
) -> *mut Stream<'a> {
314
+
unsafe {
315
+
let b = &mut *ctx;
316
+
let name_str = CStr::from_ptr(name).to_string_lossy();
317
+
318
+
match b.stream(&name_str, compress) {
319
+
Ok(stream) => Box::into_raw(Box::new(std::mem::transmute(stream))),
320
+
Err(_) => std::ptr::null_mut(),
321
+
}
322
+
}
323
+
}
324
+
325
+
#[unsafe(no_mangle)]
326
+
pub unsafe extern "C" fn bindle_stream_write(
327
+
stream: *mut Stream,
328
+
data: *const u8,
329
+
len: usize,
330
+
) -> bool {
331
+
unsafe {
332
+
let s = &mut *stream;
333
+
let chunk = std::slice::from_raw_parts(data, len);
334
+
s.write_all(chunk).is_ok()
335
+
}
336
+
}
337
+
338
+
#[unsafe(no_mangle)]
339
+
pub unsafe extern "C" fn bindle_stream_finish(stream: *mut Stream) -> bool {
340
+
// Reclaim memory from the Box and call finish()
341
+
let s = unsafe { Box::from_raw(stream) };
342
+
s.finish().is_ok()
343
+
}
+180
-1
src/lib.rs
+180
-1
src/lib.rs
···
95
95
data_end: u64,
96
96
}
97
97
98
+
pub enum Either<A, B> {
99
+
Left(A),
100
+
Right(B),
101
+
}
102
+
103
+
pub struct Stream<'a> {
104
+
pub(crate) bindle: &'a mut Bindle,
105
+
pub(crate) encoder: Option<zstd::Encoder<'a, std::fs::File>>,
106
+
pub(crate) name: String,
107
+
pub(crate) start_offset: u64,
108
+
pub(crate) uncompressed_size: u64,
109
+
}
110
+
111
+
impl<'a> std::io::Write for Stream<'a> {
112
+
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
113
+
self.write_chunk(buf)?;
114
+
Ok(buf.len())
115
+
}
116
+
117
+
fn flush(&mut self) -> io::Result<()> {
118
+
Ok(())
119
+
}
120
+
}
121
+
122
+
impl<'a> Stream<'a> {
123
+
pub fn write_chunk(&mut self, data: &[u8]) -> io::Result<()> {
124
+
self.uncompressed_size += data.len() as u64;
125
+
126
+
if let Some(encoder) = &mut self.encoder {
127
+
encoder.write_all(data)?;
128
+
} else {
129
+
self.bindle.file.write_all(data)?;
130
+
}
131
+
132
+
Ok(())
133
+
}
134
+
135
+
pub fn finish(self) -> io::Result<()> {
136
+
let compression_type = if let Some(encoder) = self.encoder {
137
+
encoder.finish()?;
138
+
1
139
+
} else {
140
+
0
141
+
};
142
+
143
+
let current_pos = self.bindle.file.stream_position()?;
144
+
let compressed_size = current_pos - self.start_offset;
145
+
146
+
// Handle 8-byte alignment padding
147
+
let pad_len = pad::<8, u64>(current_pos);
148
+
if pad_len > 0 {
149
+
self.bindle.file.write_all(&vec![0u8; pad_len as usize])?;
150
+
}
151
+
152
+
self.bindle.data_end = current_pos + pad_len;
153
+
154
+
let entry = Entry {
155
+
offset: self.start_offset.to_le_bytes(),
156
+
compressed_size: compressed_size.to_le_bytes(),
157
+
uncompressed_size: self.uncompressed_size.to_le_bytes(),
158
+
compression_type,
159
+
name_len: (self.name.len() as u16).to_le_bytes(),
160
+
..Default::default()
161
+
};
162
+
163
+
self.bindle.index.insert(self.name, entry);
164
+
Ok(())
165
+
}
166
+
}
167
+
98
168
impl Bindle {
99
169
/// Create a new bindle file, this will overwrite the existing file
100
170
pub fn create<P: AsRef<Path>>(path: P) -> io::Result<Self> {
···
131
201
let mut file = opts.open(&path)?;
132
202
file.lock_shared()?;
133
203
let len = file.metadata()?.len();
204
+
205
+
// Handle completely new/empty files
134
206
if len == 0 {
135
207
file.write_all(BNDL_MAGIC)?;
136
208
return Ok(Self {
···
142
214
});
143
215
}
144
216
217
+
// Safety check: File must be at least HEADER + FOOTER size (24 bytes)
218
+
// This prevents "attempt to subtract with overflow" when calculating footer_pos
219
+
if len < (HEADER_SIZE + FOOTER_SIZE) as u64 {
220
+
return Err(io::Error::new(
221
+
io::ErrorKind::InvalidData,
222
+
"File too small to be a valid bindle",
223
+
));
224
+
}
225
+
145
226
let mut header = [0u8; 8];
146
227
file.read_exact(&mut header)?;
147
228
if &header != BNDL_MAGIC {
···
149
230
}
150
231
151
232
let m = unsafe { Mmap::map(&file)? };
233
+
234
+
// Calculate footer position. Subtraction is now safe due to the check above.
152
235
let footer_pos = m.len() - FOOTER_SIZE;
153
236
let footer = Footer::read_from_bytes(&m[footer_pos..]).unwrap();
154
237
···
158
241
159
242
let mut cursor = data_end as usize;
160
243
for _ in 0..count {
244
+
// Ensure there is enough data left for an Entry header
245
+
if cursor + ENTRY_SIZE > footer_pos {
246
+
break;
247
+
}
248
+
161
249
let entry = Entry::read_from_bytes(&m[cursor..cursor + ENTRY_SIZE]).unwrap();
162
250
let n_start = cursor + ENTRY_SIZE;
251
+
252
+
// Validate that the filename exists within the mapped bounds
253
+
if n_start + entry.name_len() > footer_pos {
254
+
break;
255
+
}
256
+
163
257
let name =
164
258
String::from_utf8_lossy(&m[n_start..n_start + entry.name_len()]).into_owned();
165
259
index.insert(name, entry);
260
+
166
261
let total = ENTRY_SIZE + entry.name_len();
167
262
cursor += (total + (BNDL_ALIGN - 1)) & !(BNDL_ALIGN - 1);
168
263
}
···
176
271
})
177
272
}
178
273
274
+
fn should_auto_compress(&self, compress: Compress, len: usize) -> bool {
275
+
compress == Compress::Zstd || (compress == Compress::Auto && len > AUTO_COMPRESS_THRESHOLD)
276
+
}
277
+
179
278
pub fn add(&mut self, name: &str, data: &[u8], compress: Compress) -> io::Result<()> {
180
-
let compress = compress == Compress::Zstd || data.len() > AUTO_COMPRESS_THRESHOLD;
279
+
let compress = self.should_auto_compress(compress, data.len());
181
280
let (processed, c_type) = if compress {
182
281
(Cow::Owned(zstd::encode_all(data, 3)?), Compress::Zstd)
183
282
} else {
···
206
305
};
207
306
208
307
self.index.insert(name.to_string(), entry);
308
+
Ok(())
309
+
}
310
+
311
+
pub fn add_file(
312
+
&mut self,
313
+
name: &str,
314
+
path: impl AsRef<Path>,
315
+
compress: Compress,
316
+
) -> io::Result<()> {
317
+
let mut stream = self.stream(name, compress)?;
318
+
let mut src = std::fs::File::open(path)?;
319
+
std::io::copy(&mut src, &mut stream)?;
209
320
Ok(())
210
321
}
211
322
···
326
437
}
327
438
}
328
439
440
+
/// The number of entries
329
441
pub fn len(&self) -> usize {
330
442
self.index.len()
331
443
}
332
444
445
+
/// Returns true if there are no entries
333
446
pub fn is_empty(&self) -> bool {
334
447
self.index.is_empty()
335
448
}
336
449
450
+
/// Direct readonly access to the index
337
451
pub fn index(&self) -> &BTreeMap<String, Entry> {
338
452
&self.index
339
453
}
340
454
455
+
/// Clear all entries
456
+
pub fn clear(&mut self) {
457
+
self.index.clear()
458
+
}
459
+
460
+
/// Checks if an entry exists in the archive index.
461
+
pub fn exists(&self, name: &str) -> bool {
462
+
self.index.contains_key(name)
463
+
}
464
+
341
465
/// Recursively packs a directory into the archive.
342
466
pub fn pack<P: AsRef<Path>>(&mut self, src_dir: P, compress: Compress) -> io::Result<()> {
343
467
self.pack_recursive(src_dir.as_ref(), src_dir.as_ref(), compress)
···
368
492
/// Unpacks all archive entries to a destination directory.
369
493
pub fn unpack<P: AsRef<Path>>(&self, dest: P) -> io::Result<()> {
370
494
let dest_path = dest.as_ref();
495
+
if let Some(parent) = dest_path.parent() {
496
+
std::fs::create_dir_all(parent)?;
497
+
}
371
498
for (name, _) in &self.index {
372
499
if let Some(data) = self.read(name) {
373
500
let file_path = dest_path.join(name);
···
379
506
}
380
507
Ok(())
381
508
}
509
+
510
+
pub fn stream<'a>(&'a mut self, name: &str, compress: Compress) -> io::Result<Stream<'a>> {
511
+
self.file.seek(SeekFrom::Start(self.data_end))?;
512
+
let compress = self.should_auto_compress(compress, 0);
513
+
let f = self.file.try_clone()?;
514
+
let start_offset = self.data_end;
515
+
Ok(Stream {
516
+
name: name.to_string(),
517
+
bindle: self,
518
+
encoder: if compress {
519
+
Some(zstd::Encoder::new(f, 3)?)
520
+
} else {
521
+
None
522
+
},
523
+
start_offset,
524
+
uncompressed_size: 0,
525
+
})
526
+
}
382
527
}
383
528
384
529
impl Drop for Bindle {
···
588
733
fs::remove_dir_all(src_dir).ok();
589
734
fs::remove_dir_all(out_dir).ok();
590
735
fs::remove_file(bindle_path).ok();
736
+
}
737
+
738
+
#[test]
739
+
fn test_streaming_manual_chunks() {
740
+
let path = "test_stream.bindl";
741
+
let _ = std::fs::remove_file(path);
742
+
let chunk1 = b"Hello ";
743
+
let chunk2 = b"Streaming ";
744
+
let chunk3 = b"World!";
745
+
let expected = b"Hello Streaming World!";
746
+
747
+
{
748
+
let mut b = Bindle::open(path).expect("Failed to open");
749
+
// Start a stream without compression
750
+
let mut s = b
751
+
.stream("streamed_file.txt", Compress::None)
752
+
.expect("Failed to start stream");
753
+
754
+
// Write chunks manually
755
+
s.write_chunk(chunk1).unwrap();
756
+
s.write_chunk(chunk2).unwrap();
757
+
s.write_chunk(chunk3).unwrap();
758
+
759
+
s.finish().expect("Failed to finish stream");
760
+
b.save().expect("Failed to save");
761
+
}
762
+
763
+
// Verification
764
+
let b = Bindle::open(path).expect("Failed to reopen");
765
+
let result = b.read("streamed_file.txt").expect("Entry not found");
766
+
assert_eq!(result.as_ref(), expected);
767
+
assert_eq!(result.len(), expected.len());
768
+
769
+
let _ = std::fs::remove_file(path);
591
770
}
592
771
}