an efficient binary archive format
reader/writer
+131
-38
+2
-1
cbindgen.toml
+2
-1
cbindgen.toml
+13
-5
include/bindle.h
+13
-5
include/bindle.h
···
20
20
21
21
typedef struct Bindle Bindle;
22
22
23
-
typedef struct BindleStream BindleStream;
23
+
typedef struct BindleReader BindleReader;
24
+
25
+
typedef struct BindleWriter BindleWriter;
24
26
25
27
/**
26
28
* Open a bindle file from disk, the path paramter should be NUL terminated
···
97
99
bool bindle_exists(const struct Bindle *ctx, const char *name);
98
100
99
101
/**
100
-
* Create a new Stream, while the stream is active (until bindle_stream_finish is called), the
102
+
* Create a new Writer, while the stream is active (until bindle_stream_finish is called), the
101
103
* Bindle struct should not be accessed.
102
104
*/
103
-
struct BindleStream *bindle_stream_new(struct Bindle *ctx,
105
+
struct BindleWriter *bindle_writer_new(struct Bindle *ctx,
104
106
const char *name,
105
107
enum BindleCompress compress);
106
108
107
-
bool bindle_stream_write(struct BindleStream *stream, const uint8_t *data, size_t len);
109
+
bool bindle_writer_write(struct BindleWriter *stream, const uint8_t *data, size_t len);
108
110
109
-
bool bindle_stream_finish(struct BindleStream *stream);
111
+
bool bindle_writer_finish(struct BindleWriter *stream);
112
+
113
+
struct BindleReader *bindle_reader_new(const struct Bindle *ctx, const char *name);
114
+
115
+
ptrdiff_t bindle_reader_read(struct BindleReader *reader, uint8_t *buffer, size_t buffer_len);
116
+
117
+
void bindle_reader_free(struct BindleReader *reader);
110
118
111
119
#endif /* BINDLE_H */
+55
-9
src/ffi.rs
+55
-9
src/ffi.rs
···
1
1
use std::alloc::{Layout, dealloc};
2
2
use std::ffi::CStr;
3
-
use std::io::Write;
3
+
use std::io::{Read, Write};
4
4
use std::mem;
5
5
use std::os::raw::c_char;
6
6
use std::slice;
7
7
8
-
use crate::{Bindle, Compress, Stream};
8
+
use crate::{Bindle, Compress, Reader, Writer};
9
9
10
10
/// Open a bindle file from disk, the path paramter should be NUL terminated
11
11
#[unsafe(no_mangle)]
···
305
305
b.exists(name_str)
306
306
}
307
307
308
-
/// Create a new Stream, while the stream is active (until bindle_stream_finish is called), the
308
+
/// Create a new Writer, while the stream is active (until bindle_stream_finish is called), the
309
309
/// Bindle struct should not be accessed.
310
310
#[unsafe(no_mangle)]
311
-
pub unsafe extern "C" fn bindle_stream_new<'a>(
311
+
pub unsafe extern "C" fn bindle_writer_new<'a>(
312
312
ctx: *mut Bindle,
313
313
name: *const c_char,
314
314
compress: Compress,
315
-
) -> *mut Stream<'a> {
315
+
) -> *mut Writer<'a> {
316
316
unsafe {
317
317
let b = &mut *ctx;
318
318
let name_str = CStr::from_ptr(name).to_string_lossy();
319
319
320
-
match b.stream(&name_str, compress) {
320
+
match b.writer(&name_str, compress) {
321
321
Ok(stream) => Box::into_raw(Box::new(std::mem::transmute(stream))),
322
322
Err(_) => std::ptr::null_mut(),
323
323
}
···
325
325
}
326
326
327
327
#[unsafe(no_mangle)]
328
-
pub unsafe extern "C" fn bindle_stream_write(
329
-
stream: *mut Stream,
328
+
pub unsafe extern "C" fn bindle_writer_write(
329
+
stream: *mut Writer,
330
330
data: *const u8,
331
331
len: usize,
332
332
) -> bool {
···
338
338
}
339
339
340
340
#[unsafe(no_mangle)]
341
-
pub unsafe extern "C" fn bindle_stream_finish(stream: *mut Stream) -> bool {
341
+
pub unsafe extern "C" fn bindle_writer_finish(stream: *mut Writer) -> bool {
342
342
// Reclaim memory from the Box and call finish()
343
343
let s = unsafe { Box::from_raw(stream) };
344
344
s.finish().is_ok()
345
345
}
346
+
347
+
#[unsafe(no_mangle)]
348
+
pub unsafe extern "C" fn bindle_reader_new<'a>(
349
+
ctx: *const Bindle,
350
+
name: *const c_char,
351
+
) -> *mut Reader<'a> {
352
+
if ctx.is_null() || name.is_null() {
353
+
return std::ptr::null_mut();
354
+
}
355
+
356
+
let b = unsafe { &*ctx };
357
+
let name_str = unsafe { CStr::from_ptr(name).to_string_lossy() };
358
+
359
+
match b.reader(&name_str) {
360
+
Ok(reader) => Box::into_raw(Box::new(reader)),
361
+
Err(_) => std::ptr::null_mut(),
362
+
}
363
+
}
364
+
365
+
#[unsafe(no_mangle)]
366
+
pub unsafe extern "C" fn bindle_reader_read(
367
+
reader: *mut Reader,
368
+
buffer: *mut u8,
369
+
buffer_len: usize,
370
+
) -> isize {
371
+
if reader.is_null() || buffer.is_null() {
372
+
return -1;
373
+
}
374
+
375
+
let r = unsafe { &mut *reader };
376
+
let out_slice = unsafe { slice::from_raw_parts_mut(buffer, buffer_len) };
377
+
378
+
match r.read(out_slice) {
379
+
Ok(n) => n as isize,
380
+
Err(_) => -1,
381
+
}
382
+
}
383
+
384
+
#[unsafe(no_mangle)]
385
+
pub unsafe extern "C" fn bindle_reader_free(reader: *mut Reader) {
386
+
if !reader.is_null() {
387
+
unsafe {
388
+
drop(Box::from_raw(reader));
389
+
}
390
+
}
391
+
}
+61
-23
src/lib.rs
+61
-23
src/lib.rs
···
3
3
use std::borrow::Cow;
4
4
use std::collections::BTreeMap;
5
5
use std::fs::{File, OpenOptions};
6
-
use std::io::{self, Read, Seek, SeekFrom, Write};
6
+
use std::io::{self, BufReader, Read, Seek, SeekFrom, Write};
7
7
use std::path::{Path, PathBuf};
8
8
use zerocopy::{FromBytes, Immutable, IntoBytes, Unaligned};
9
9
···
100
100
Right(B),
101
101
}
102
102
103
-
pub struct Stream<'a> {
103
+
pub struct Reader<'a> {
104
+
decoder: Either<zstd::Decoder<'static, BufReader<io::Cursor<&'a [u8]>>>, io::Cursor<&'a [u8]>>,
105
+
}
106
+
107
+
impl<'a> Read for Reader<'a> {
108
+
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
109
+
match &mut self.decoder {
110
+
Either::Left(x) => x.read(buf),
111
+
Either::Right(x) => x.read(buf),
112
+
}
113
+
}
114
+
}
115
+
116
+
// Note: Seeking is only supported for uncompressed entries in this simple implementation.
117
+
// Seeking in compressed streams requires a frame-aware decoder.
118
+
impl<'a> Seek for Reader<'a> {
119
+
fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
120
+
match &mut self.decoder {
121
+
Either::Left(_) => Err(io::Error::new(
122
+
io::ErrorKind::Unsupported,
123
+
"Seeking not supported on compressed streams",
124
+
)),
125
+
Either::Right(x) => x.seek(pos),
126
+
}
127
+
}
128
+
}
129
+
130
+
pub struct Writer<'a> {
104
131
pub(crate) bindle: &'a mut Bindle,
105
132
pub(crate) encoder: Option<zstd::Encoder<'a, std::fs::File>>,
106
133
pub(crate) name: String,
···
108
135
pub(crate) uncompressed_size: u64,
109
136
}
110
137
111
-
impl<'a> std::io::Write for Stream<'a> {
138
+
impl<'a> std::io::Write for Writer<'a> {
112
139
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
113
140
self.write_chunk(buf)?;
114
141
Ok(buf.len())
···
119
146
}
120
147
}
121
148
122
-
impl<'a> Stream<'a> {
149
+
impl<'a> Writer<'a> {
123
150
pub fn write_chunk(&mut self, data: &[u8]) -> io::Result<()> {
124
151
self.uncompressed_size += data.len() as u64;
125
152
···
276
303
}
277
304
278
305
pub fn add(&mut self, name: &str, data: &[u8], compress: Compress) -> io::Result<()> {
279
-
let mut stream = self.stream(name, compress)?;
306
+
let mut stream = self.writer(name, compress)?;
280
307
stream.write_all(data)?;
281
308
stream.finish()?;
282
309
Ok(())
···
288
315
path: impl AsRef<Path>,
289
316
compress: Compress,
290
317
) -> io::Result<()> {
291
-
let mut stream = self.stream(name, compress)?;
318
+
let mut stream = self.writer(name, compress)?;
292
319
let mut src = std::fs::File::open(path)?;
293
320
std::io::copy(&mut src, &mut stream)?;
294
321
Ok(())
···
302
329
for (name, entry) in &self.index {
303
330
self.file.write_all(entry.as_bytes())?;
304
331
self.file.write_all(name.as_bytes())?;
305
-
let pad = pad::<BNDL_ALIGN, usize>(ENTRY_SIZE + name.len()); // (BNDL_ALIGN - ((ENTRY_SIZE + name.len()) % BNDL_ALIGN)) % BNDL_ALIGN;
332
+
let pad = pad::<BNDL_ALIGN, usize>(ENTRY_SIZE + name.len());
306
333
if pad > 0 {
307
334
self.file.write_all(&vec![0u8; pad])?;
308
335
}
···
411
438
}
412
439
}
413
440
414
-
pub fn read_to<W: std::io::Write>(&self, name: &str, mut w: W) -> std::io::Result<()> {
441
+
/// Read to an `std::io::Write`
442
+
pub fn read_to<W: std::io::Write>(&self, name: &str, mut w: W) -> std::io::Result<u64> {
443
+
std::io::copy(&mut self.reader(name)?, &mut w)
444
+
}
445
+
446
+
// Returns a seekable reader for an entry.
447
+
/// If compressed, it provides a transparently decompressing stream.
448
+
pub fn reader<'a>(&'a self, name: &str) -> io::Result<Reader<'a>> {
415
449
let entry = self
416
450
.index
417
451
.get(name)
418
-
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Invalid entry"))?;
452
+
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "Entry not found"))?;
453
+
454
+
let start = entry.offset() as usize;
455
+
let end = start + entry.compressed_size() as usize;
419
456
let mmap = self
420
457
.mmap
421
458
.as_ref()
422
459
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Missing mmap"))?;
423
-
let data = mmap
424
-
.get(entry.offset() as usize..(entry.offset() + entry.compressed_size()) as usize)
425
-
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Invalid mmap offset"))?;
460
+
let data_slice = &mmap[start..end];
426
461
427
-
if entry.compression_type == Compress::Zstd as u8 {
428
-
std::io::copy(
429
-
&mut zstd::Decoder::new(data)
430
-
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))?,
431
-
&mut w,
432
-
)?;
462
+
let cursor = io::Cursor::new(data_slice);
463
+
464
+
if entry.compression_type == 1 {
465
+
// Zstd streaming decoder
466
+
let decoder = zstd::Decoder::new(cursor)?;
467
+
Ok(Reader {
468
+
decoder: Either::Left(decoder),
469
+
})
433
470
} else {
434
-
w.write_all(&data)?;
471
+
Ok(Reader {
472
+
decoder: Either::Right(cursor),
473
+
})
435
474
}
436
-
Ok(())
437
475
}
438
476
439
477
/// The number of entries
···
506
544
Ok(())
507
545
}
508
546
509
-
pub fn stream<'a>(&'a mut self, name: &str, compress: Compress) -> io::Result<Stream<'a>> {
547
+
pub fn writer<'a>(&'a mut self, name: &str, compress: Compress) -> io::Result<Writer<'a>> {
510
548
self.file.seek(SeekFrom::Start(self.data_end))?;
511
549
let compress = self.should_auto_compress(compress, 0);
512
550
let f = self.file.try_clone()?;
513
551
let start_offset = self.data_end;
514
-
Ok(Stream {
552
+
Ok(Writer {
515
553
name: name.to_string(),
516
554
bindle: self,
517
555
encoder: if compress {
···
747
785
let mut b = Bindle::open(path).expect("Failed to open");
748
786
// Start a stream without compression
749
787
let mut s = b
750
-
.stream("streamed_file.txt", Compress::None)
788
+
.writer("streamed_file.txt", Compress::None)
751
789
.expect("Failed to start stream");
752
790
753
791
// Write chunks manually