Bytesrw adapter for Eio
ocaml
codec
Squashed 'ocaml-bytesrw-eio/' content from commit 9b537a9 git-subtree-split: 9b537a90331d85f9cf3a268bf408286cc4db9e8b
+506
+17
.gitignore
+17
.gitignore
+1
.ocamlformat
+1
.ocamlformat
···
1
+
version = 0.28.1
+49
.tangled/workflows/build.yml
+49
.tangled/workflows/build.yml
···
1
+
when:
2
+
- event: ["push", "pull_request"]
3
+
branch: ["main"]
4
+
5
+
engine: nixery
6
+
7
+
dependencies:
8
+
nixpkgs:
9
+
- shell
10
+
- stdenv
11
+
- findutils
12
+
- binutils
13
+
- libunwind
14
+
- ncurses
15
+
- opam
16
+
- git
17
+
- gawk
18
+
- gnupatch
19
+
- gnum4
20
+
- gnumake
21
+
- gnutar
22
+
- gnused
23
+
- gnugrep
24
+
- diffutils
25
+
- gzip
26
+
- bzip2
27
+
- gcc
28
+
- ocaml
29
+
30
+
steps:
31
+
- name: opam
32
+
command: |
33
+
opam init --disable-sandboxing -a -y
34
+
- name: switch
35
+
command: |
36
+
opam install . --confirm-level=unsafe-yes --deps-only
37
+
- name: build
38
+
command: |
39
+
opam exec -- dune build
40
+
- name: switch-test
41
+
command: |
42
+
opam install . --confirm-level=unsafe-yes --deps-only --with-test
43
+
- name: test
44
+
command: |
45
+
opam exec -- dune runtest --verbose
46
+
- name: doc
47
+
command: |
48
+
opam install -y odoc
49
+
opam exec -- dune build @doc
+15
LICENSE.md
+15
LICENSE.md
···
1
+
ISC License
2
+
3
+
Copyright (c) 2025 Anil Madhavapeddy <anil@recoil.org>
4
+
5
+
Permission to use, copy, modify, and distribute this software for any
6
+
purpose with or without fee is hereby granted, provided that the above
7
+
copyright notice and this permission notice appear in all copies.
8
+
9
+
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10
+
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11
+
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12
+
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13
+
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14
+
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15
+
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+53
README.md
+53
README.md
···
1
+
# bytesrw-eio - OCaml Bytesrw adapters for Eio
2
+
3
+
This OCaml library provides adapters to create `Bytesrw.Bytes.Reader.t` and
4
+
`Bytesrw.Bytes.Writer.t` from Eio flows, mirroring the API of `Bytesrw_unix`
5
+
for Eio's effect-based I/O.
6
+
7
+
## Usage
8
+
9
+
```ocaml
10
+
open Eio.Std
11
+
12
+
(* Create a reader from an Eio flow *)
13
+
let read_from_flow flow =
14
+
let reader = Bytesrw_eio.bytes_reader_of_flow flow in
15
+
(* Use reader with Bytesrw decoders *)
16
+
reader
17
+
18
+
(* Create a writer to an Eio flow *)
19
+
let write_to_flow flow =
20
+
let writer = Bytesrw_eio.bytes_writer_of_flow flow in
21
+
(* Use writer with Bytesrw encoders *)
22
+
writer
23
+
```
24
+
25
+
For custom slice sizes:
26
+
27
+
```ocaml
28
+
(* Specify custom slice length for reading *)
29
+
let reader = Bytesrw_eio.bytes_reader_of_flow ~slice_length:4096 flow in
30
+
31
+
(* Specify custom slice length for writing *)
32
+
let writer = Bytesrw_eio.bytes_writer_of_flow ~slice_length:4096 flow in
33
+
()
34
+
```
35
+
36
+
## Installation
37
+
38
+
```
39
+
opam install bytesrw-eio
40
+
```
41
+
42
+
## Documentation
43
+
44
+
API documentation is available via:
45
+
46
+
```
47
+
opam install bytesrw-eio
48
+
odig doc bytesrw-eio
49
+
```
50
+
51
+
## License
52
+
53
+
ISC
+34
bytesrw-eio.opam
+34
bytesrw-eio.opam
···
1
+
# This file is generated by dune, edit dune-project instead
2
+
opam-version: "2.0"
3
+
synopsis: "Bytesrw readers and writers for Eio"
4
+
description:
5
+
"Provides Bytesrw.Bytes.Reader and Writer adapters for Eio Flows"
6
+
maintainer: ["Anil Madhavapeddy <anil@recoil.org>"]
7
+
authors: ["Anil Madhavapeddy"]
8
+
license: "ISC"
9
+
homepage: "https://tangled.org/anil.recoil.org/ocaml-bytesrw-eio"
10
+
bug-reports: "https://tangled.org/anil.recoil.org/ocaml-bytesrw-eio/issues"
11
+
depends: [
12
+
"dune" {>= "3.18"}
13
+
"ocaml" {>= "5.0"}
14
+
"bytesrw" {>= "0.2"}
15
+
"eio" {>= "1.0"}
16
+
"odoc" {with-doc}
17
+
"alcotest" {with-test & >= "1.7.0"}
18
+
"eio_main" {with-test}
19
+
]
20
+
build: [
21
+
["dune" "subst"] {dev}
22
+
[
23
+
"dune"
24
+
"build"
25
+
"-p"
26
+
name
27
+
"-j"
28
+
jobs
29
+
"@install"
30
+
"@runtest" {with-test}
31
+
"@doc" {with-doc}
32
+
]
33
+
]
34
+
x-maintenance-intent: ["(latest)"]
+24
dune-project
+24
dune-project
···
1
+
(lang dune 3.18)
2
+
3
+
(name bytesrw-eio)
4
+
5
+
(generate_opam_files true)
6
+
7
+
(license ISC)
8
+
(authors "Anil Madhavapeddy")
9
+
(homepage "https://tangled.org/anil.recoil.org/ocaml-bytesrw-eio")
10
+
(maintainers "Anil Madhavapeddy <anil@recoil.org>")
11
+
(bug_reports "https://tangled.org/anil.recoil.org/ocaml-bytesrw-eio/issues")
12
+
(maintenance_intent "(latest)")
13
+
14
+
(package
15
+
(name bytesrw-eio)
16
+
(synopsis "Bytesrw readers and writers for Eio")
17
+
(description "Provides Bytesrw.Bytes.Reader and Writer adapters for Eio Flows")
18
+
(depends
19
+
(ocaml (>= 5.0))
20
+
(bytesrw (>= 0.2))
21
+
(eio (>= 1.0))
22
+
(odoc :with-doc)
23
+
(alcotest (and :with-test (>= 1.7.0)))
24
+
(eio_main :with-test)))
+57
src/bytesrw_eio.ml
+57
src/bytesrw_eio.ml
···
1
+
(*---------------------------------------------------------------------------
2
+
Copyright (c) 2025 Anil Madhavapeddy <anil@recoil.org>. All rights reserved.
3
+
SPDX-License-Identifier: ISC
4
+
---------------------------------------------------------------------------*)
5
+
6
+
(** Bytesrw adapters for Eio
7
+
8
+
This module provides adapters to create {!Bytesrw.Bytes.Reader.t} and
9
+
{!Bytesrw.Bytes.Writer.t} from Eio flows, mirroring the API of
10
+
{!Bytesrw_unix} for Eio's effect-based I/O. *)
11
+
12
+
open Bytesrw
13
+
14
+
(** Create a [Bytes.Reader.t] from an Eio source flow.
15
+
16
+
Reads directly from the flow without intermediate buffering.
17
+
18
+
@param slice_length
19
+
Maximum bytes per slice (default: 65536, which is
20
+
{!Bytes.Slice.unix_io_buffer_size}) *)
21
+
let bytes_reader_of_flow ?(slice_length = Bytes.Slice.unix_io_buffer_size)
22
+
(flow : _ Eio.Flow.source) : Bytes.Reader.t =
23
+
let buf_size = Bytes.Slice.check_length slice_length in
24
+
let read () =
25
+
let cstruct = Cstruct.create buf_size in
26
+
match Eio.Flow.single_read flow cstruct with
27
+
| 0 -> Bytes.Slice.eod
28
+
| count ->
29
+
let data_cs = Cstruct.sub cstruct 0 count in
30
+
let buf = Cstruct.to_bytes data_cs in
31
+
Bytes.Slice.make buf ~first:0 ~length:count
32
+
| exception End_of_file -> Bytes.Slice.eod
33
+
in
34
+
Bytes.Reader.make ~slice_length read
35
+
36
+
(** Create a [Bytes.Writer.t] from an Eio sink flow.
37
+
38
+
Writes directly to the flow without intermediate buffering.
39
+
40
+
@param slice_length
41
+
Suggested slice length for upstream (default: 65536, which is
42
+
{!Bytes.Slice.unix_io_buffer_size}) *)
43
+
let bytes_writer_of_flow ?(slice_length = Bytes.Slice.unix_io_buffer_size)
44
+
(flow : _ Eio.Flow.sink) : Bytes.Writer.t =
45
+
let rec write slice =
46
+
if Bytes.Slice.is_eod slice then ()
47
+
else begin
48
+
let bytes = Bytes.Slice.bytes slice in
49
+
let first = Bytes.Slice.first slice in
50
+
let length = Bytes.Slice.length slice in
51
+
let cstruct = Cstruct.of_bytes ~off:first ~len:length bytes in
52
+
match Eio.Flow.single_write flow [ cstruct ] with
53
+
| count when count = length -> ()
54
+
| count -> write (Option.get (Bytes.Slice.drop count slice))
55
+
end
56
+
in
57
+
Bytes.Writer.make ~slice_length write
+33
src/bytesrw_eio.mli
+33
src/bytesrw_eio.mli
···
1
+
(*---------------------------------------------------------------------------
2
+
Copyright (c) 2025 Anil Madhavapeddy <anil@recoil.org>. All rights reserved.
3
+
SPDX-License-Identifier: ISC
4
+
---------------------------------------------------------------------------*)
5
+
6
+
(** Bytesrw adapters for Eio
7
+
8
+
This module provides adapters to create {!Bytesrw.Bytes.Reader.t} and
9
+
{!Bytesrw.Bytes.Writer.t} from Eio flows, mirroring the API of
10
+
{!Bytesrw_unix} for Eio's effect-based I/O.
11
+
12
+
Unlike the Buf_read/Buf_write wrappers, these adapters read and write
13
+
directly to the flow, allowing bytesrw to handle its own buffering. *)
14
+
15
+
(** {1 Readers} *)
16
+
17
+
val bytes_reader_of_flow :
18
+
?slice_length:int -> _ Eio.Flow.source -> Bytesrw.Bytes.Reader.t
19
+
(** [bytes_reader_of_flow flow] creates a reader from an Eio source flow.
20
+
21
+
Reads directly from the flow without intermediate buffering.
22
+
23
+
@param slice_length Maximum bytes per slice (default: 65536) *)
24
+
25
+
(** {1 Writers} *)
26
+
27
+
val bytes_writer_of_flow :
28
+
?slice_length:int -> _ Eio.Flow.sink -> Bytesrw.Bytes.Writer.t
29
+
(** [bytes_writer_of_flow flow] creates a writer from an Eio sink flow.
30
+
31
+
Writes directly to the flow without intermediate buffering.
32
+
33
+
@param slice_length Suggested slice length for upstream (default: 65536) *)
+4
src/dune
+4
src/dune
+3
test/dune
+3
test/dune
+216
test/test_bytesrw_eio.ml
+216
test/test_bytesrw_eio.ml
···
1
+
(*---------------------------------------------------------------------------
2
+
Copyright (c) 2025 Anil Madhavapeddy <anil@recoil.org>. All rights reserved.
3
+
SPDX-License-Identifier: ISC
4
+
---------------------------------------------------------------------------*)
5
+
6
+
(* Test reading from a mock flow *)
7
+
let test_reader_basic () =
8
+
Eio_main.run @@ fun _env ->
9
+
let test_data = "Hello, World!" in
10
+
let flow = Eio.Flow.string_source test_data in
11
+
let reader = Bytesrw_eio.bytes_reader_of_flow flow in
12
+
13
+
(* Read first slice *)
14
+
let slice1 = Bytesrw.Bytes.Reader.read reader in
15
+
Alcotest.(check bool)
16
+
"slice is not eod" false
17
+
(Bytesrw.Bytes.Slice.is_eod slice1);
18
+
19
+
let read_data =
20
+
Bytes.sub_string
21
+
(Bytesrw.Bytes.Slice.bytes slice1)
22
+
(Bytesrw.Bytes.Slice.first slice1)
23
+
(Bytesrw.Bytes.Slice.length slice1)
24
+
in
25
+
Alcotest.(check string) "data matches" test_data read_data;
26
+
27
+
(* Next read should be eod *)
28
+
let slice2 = Bytesrw.Bytes.Reader.read reader in
29
+
Alcotest.(check bool)
30
+
"second read is eod" true
31
+
(Bytesrw.Bytes.Slice.is_eod slice2)
32
+
33
+
(* Test reading with custom slice length *)
34
+
let test_reader_custom_slice_length () =
35
+
Eio_main.run @@ fun _env ->
36
+
let test_data = "Hello, World!" in
37
+
let flow = Eio.Flow.string_source test_data in
38
+
let slice_length = 5 in
39
+
let reader = Bytesrw_eio.bytes_reader_of_flow ~slice_length flow in
40
+
41
+
(* Read should respect slice_length as maximum *)
42
+
let slice = Bytesrw.Bytes.Reader.read reader in
43
+
Alcotest.(check bool)
44
+
"slice length <= custom size" true
45
+
(Bytesrw.Bytes.Slice.length slice <= slice_length)
46
+
47
+
(* Test reading empty flow *)
48
+
let test_reader_empty () =
49
+
Eio_main.run @@ fun _env ->
50
+
let flow = Eio.Flow.string_source "" in
51
+
let reader = Bytesrw_eio.bytes_reader_of_flow flow in
52
+
53
+
let slice = Bytesrw.Bytes.Reader.read reader in
54
+
Alcotest.(check bool)
55
+
"empty flow returns eod" true
56
+
(Bytesrw.Bytes.Slice.is_eod slice)
57
+
58
+
(* Test writing to a mock flow *)
59
+
let test_writer_basic () =
60
+
Eio_main.run @@ fun _env ->
61
+
let buf = Buffer.create 100 in
62
+
let flow = Eio.Flow.buffer_sink buf in
63
+
let writer = Bytesrw_eio.bytes_writer_of_flow flow in
64
+
65
+
let test_data = "Hello, World!" in
66
+
let bytes = Bytes.of_string test_data in
67
+
let slice =
68
+
Bytesrw.Bytes.Slice.make bytes ~first:0 ~length:(Bytes.length bytes)
69
+
in
70
+
71
+
Bytesrw.Bytes.Writer.write writer slice;
72
+
73
+
let written = Buffer.contents buf in
74
+
Alcotest.(check string) "written data matches" test_data written
75
+
76
+
(* Test writing with custom slice length *)
77
+
let test_writer_custom_slice_length () =
78
+
Eio_main.run @@ fun _env ->
79
+
let buf = Buffer.create 100 in
80
+
let flow = Eio.Flow.buffer_sink buf in
81
+
let slice_length = 8 in
82
+
let writer = Bytesrw_eio.bytes_writer_of_flow ~slice_length flow in
83
+
84
+
let test_data = "Hello, World!" in
85
+
let bytes = Bytes.of_string test_data in
86
+
let slice =
87
+
Bytesrw.Bytes.Slice.make bytes ~first:0 ~length:(Bytes.length bytes)
88
+
in
89
+
90
+
Bytesrw.Bytes.Writer.write writer slice;
91
+
92
+
let written = Buffer.contents buf in
93
+
Alcotest.(check string)
94
+
"written data matches regardless of slice_length" test_data written
95
+
96
+
(* Test writing eod slice (should be no-op) *)
97
+
let test_writer_eod () =
98
+
Eio_main.run @@ fun _env ->
99
+
let buf = Buffer.create 100 in
100
+
let flow = Eio.Flow.buffer_sink buf in
101
+
let writer = Bytesrw_eio.bytes_writer_of_flow flow in
102
+
103
+
Bytesrw.Bytes.Writer.write writer Bytesrw.Bytes.Slice.eod;
104
+
105
+
let written = Buffer.contents buf in
106
+
Alcotest.(check string) "eod writes nothing" "" written
107
+
108
+
(* Test writing partial slice *)
109
+
let test_writer_partial_slice () =
110
+
Eio_main.run @@ fun _env ->
111
+
let buf = Buffer.create 100 in
112
+
let flow = Eio.Flow.buffer_sink buf in
113
+
let writer = Bytesrw_eio.bytes_writer_of_flow flow in
114
+
115
+
let test_data = "Hello, World!" in
116
+
let bytes = Bytes.of_string test_data in
117
+
(* Write only "World" *)
118
+
let slice = Bytesrw.Bytes.Slice.make bytes ~first:7 ~length:5 in
119
+
120
+
Bytesrw.Bytes.Writer.write writer slice;
121
+
122
+
let written = Buffer.contents buf in
123
+
Alcotest.(check string) "partial slice written" "World" written
124
+
125
+
(* Test multiple reads to ensure data isolation - buffers from previous reads
126
+
should not be corrupted by subsequent reads *)
127
+
let test_reader_multiple_reads () =
128
+
Eio_main.run @@ fun _env ->
129
+
let test_data = "ABCDEFGHIJ" in
130
+
(* 10 bytes *)
131
+
let flow = Eio.Flow.string_source test_data in
132
+
let reader = Bytesrw_eio.bytes_reader_of_flow ~slice_length:5 flow in
133
+
134
+
(* Read first 5 bytes *)
135
+
let slice1 = Bytesrw.Bytes.Reader.read reader in
136
+
let bytes1 = Bytesrw.Bytes.Slice.bytes slice1 in
137
+
let data1 =
138
+
Bytes.sub_string bytes1
139
+
(Bytesrw.Bytes.Slice.first slice1)
140
+
(Bytesrw.Bytes.Slice.length slice1)
141
+
in
142
+
143
+
(* Read next 5 bytes *)
144
+
let slice2 = Bytesrw.Bytes.Reader.read reader in
145
+
let data2 =
146
+
Bytes.sub_string
147
+
(Bytesrw.Bytes.Slice.bytes slice2)
148
+
(Bytesrw.Bytes.Slice.first slice2)
149
+
(Bytesrw.Bytes.Slice.length slice2)
150
+
in
151
+
152
+
(* Critical test: verify first read's data is STILL intact after second read
153
+
This would fail if we were reusing buffers or if Cstruct.to_bytes created a view *)
154
+
let data1_check =
155
+
Bytes.sub_string bytes1
156
+
(Bytesrw.Bytes.Slice.first slice1)
157
+
(Bytesrw.Bytes.Slice.length slice1)
158
+
in
159
+
160
+
Alcotest.(check string) "first read" "ABCDE" data1;
161
+
Alcotest.(check string) "second read" "FGHIJ" data2;
162
+
Alcotest.(check string)
163
+
"first read still intact after second" "ABCDE" data1_check
164
+
165
+
(* Test round-trip: write then read *)
166
+
let test_roundtrip () =
167
+
Eio_main.run @@ fun _env ->
168
+
let test_data = "Round-trip test data" in
169
+
170
+
(* Write to buffer *)
171
+
let buf = Buffer.create 100 in
172
+
let write_flow = Eio.Flow.buffer_sink buf in
173
+
let writer = Bytesrw_eio.bytes_writer_of_flow write_flow in
174
+
175
+
let bytes = Bytes.of_string test_data in
176
+
let slice =
177
+
Bytesrw.Bytes.Slice.make bytes ~first:0 ~length:(Bytes.length bytes)
178
+
in
179
+
Bytesrw.Bytes.Writer.write writer slice;
180
+
181
+
(* Read back from buffer *)
182
+
let read_flow = Eio.Flow.string_source (Buffer.contents buf) in
183
+
let reader = Bytesrw_eio.bytes_reader_of_flow read_flow in
184
+
185
+
let read_slice = Bytesrw.Bytes.Reader.read reader in
186
+
let read_data =
187
+
Bytes.sub_string
188
+
(Bytesrw.Bytes.Slice.bytes read_slice)
189
+
(Bytesrw.Bytes.Slice.first read_slice)
190
+
(Bytesrw.Bytes.Slice.length read_slice)
191
+
in
192
+
193
+
Alcotest.(check string) "round-trip data matches" test_data read_data
194
+
195
+
let () =
196
+
Alcotest.run "Bytesrw_eio"
197
+
[
198
+
( "reader",
199
+
[
200
+
Alcotest.test_case "basic read" `Quick test_reader_basic;
201
+
Alcotest.test_case "custom slice length" `Quick
202
+
test_reader_custom_slice_length;
203
+
Alcotest.test_case "empty flow" `Quick test_reader_empty;
204
+
Alcotest.test_case "multiple reads data isolation" `Quick
205
+
test_reader_multiple_reads;
206
+
] );
207
+
( "writer",
208
+
[
209
+
Alcotest.test_case "basic write" `Quick test_writer_basic;
210
+
Alcotest.test_case "custom slice length" `Quick
211
+
test_writer_custom_slice_length;
212
+
Alcotest.test_case "eod write" `Quick test_writer_eod;
213
+
Alcotest.test_case "partial slice" `Quick test_writer_partial_slice;
214
+
] );
215
+
("integration", [ Alcotest.test_case "round-trip" `Quick test_roundtrip ]);
216
+
]