s3#

s3 is an OCaml library for AWS S3 and S3-compatible APIs.

It includes:

streaming upload and download
list, head, get, put, copy, delete, and multipart operations
AWS SigV4 signing
support for custom endpoints and bucket addressing styles
a small s3-cli binary for quick testing

Requirements#

OCaml 5.1+
Eio runtime

Main dependencies used by the library:

eio
tls
xmlm
simdjsont

Quick Start#

Environment#

The CLI uses these environment variables:

S3_BUCKET
S3_REGION
S3_ACCESS_KEY_ID
S3_SECRET_ACCESS_KEY
S3_ENDPOINT

Optional:

S3_SESSION_TOKEN
S3_ADDRESSING_STYLE with path or virtual
S3_MULTIPART_THRESHOLD in bytes
S3_MULTIPART_PART_SIZE in bytes

Example for Backblaze B2:

export S3_BUCKET=my-bucket
export S3_REGION=eu-central-003
export S3_ACCESS_KEY_ID=...
export S3_SECRET_ACCESS_KEY=...
export S3_ENDPOINT=s3.......backblazeb2.com
export S3_ADDRESSING_STYLE=virtual

Example for AWS S3:

export S3_BUCKET=my-bucket
export S3_REGION=eu-west-1
export S3_ACCESS_KEY_ID=...
export S3_SECRET_ACCESS_KEY=...
export S3_ENDPOINT=s3........amazonaws.com
export S3_ADDRESSING_STYLE=virtual

CLI#

List objects:

dune exec -- s3-cli list
dune exec -- s3-cli list hcs

Upload a file:

dune exec -- s3-cli put ./local-file.txt folder/remote-file.txt

put automatically switches to multipart upload for large files.

To force multipart for testing, set a low threshold:

export S3_MULTIPART_THRESHOLD=5242880
export S3_MULTIPART_PART_SIZE=5242880
dune exec -- s3-cli put ./large-file.bin test/large-file.bin

Download a file:

dune exec -- s3-cli get folder/remote-file.txt ./local-file.txt

Delete an object:

dune exec -- s3-cli delete folder/remote-file.txt

Show request/signing debug output:

dune exec -- s3-cli list --debug hcs

You can also use --list, --put, --get, and --delete as shorthand entry forms.

Library Usage#

Create a client:

let region = Result.get_ok (S3.Region.of_string "eu-west-1")

let endpoint =
  S3.Endpoint.custom
    ~scheme:"https"
    ~host:"s3.eu-west-1.amazonaws.com"
    ~addressing_style:S3.Endpoint.Virtual_hosted
    ~signing_region:region
    ()

let credentials =
  S3.Credentials.Source.static
    {
      S3.Credentials.access_key_id = "...";
      secret_access_key = "...";
      session_token = None;
      expiration = None;
    }

let client = S3.create ~endpoint ~credentials ()

List objects:

let list env client bucket prefix =
  S3.list_objects_v2 ~env client
    {
      S3.List_objects_v2.bucket = bucket;
      prefix;
      delimiter = None;
      continuation_token = None;
      start_after = None;
      max_keys = None;
    }

Upload a string:

let put env client bucket key contents =
  S3.put_object ~env client
    {
      S3.Put_object.bucket = bucket;
      key;
      body = S3.Stream.of_string contents;
      content_type = Some "text/plain";
      metadata = [];
      checksum = None;
    }

Multipart upload:

let multipart_put env client bucket key part1 part2 =
  match
    S3.create_multipart_upload ~env client
      {
        S3.Multipart.bucket = bucket;
        key;
        content_type = Some "application/octet-stream";
        metadata = [];
      }
  with
  | Error err -> Error err
  | Ok upload -> (
      match
        S3.upload_part ~env client
          {
            S3.Multipart.upload;
            part_number = 1;
            body = S3.Stream.of_string part1;
            checksum = None;
          }
      with
      | Error err ->
          ignore (S3.abort_multipart_upload ~env client upload);
          Error err
      | Ok part1_response -> (
          match
            S3.upload_part ~env client
              {
                S3.Multipart.upload;
                part_number = 2;
                body = S3.Stream.of_string part2;
                checksum = None;
              }
          with
          | Error err ->
              ignore (S3.abort_multipart_upload ~env client upload);
              Error err
          | Ok part2_response ->
              let open S3.Multipart in
              let parts =
                [ { part_number = 1; etag = Option.get part1_response.etag; checksum = None };
                  { part_number = 2; etag = Option.get part2_response.etag; checksum = None } ]
              in
              S3.complete_multipart_upload ~env client { upload; parts }))

Download to a buffer:

let get_to_string env client bucket key =
  let buffer = Buffer.create 1024 in
  let sink = S3.Stream.to_buffer buffer in
  match
    S3.get_object ~env client
      {
        S3.Get_object.bucket = bucket;
        key;
        range = None;
        if_match = None;
        if_none_match = None;
      }
      ~sink
  with
  | Ok _ -> Ok (Buffer.contents buffer)
  | Error err -> Error err

Delete an object:

let delete env client bucket key =
  S3.delete_object ~env client
    { S3.Delete_object.bucket = bucket; key; version_id = None }

Addressing Style#

Use path when the bucket is part of the URL path:

https://s3.example.com/my-bucket/object.txt

Use virtual when the bucket is part of the host:

https://my-bucket.s3.example.com/object.txt

Some S3-compatible providers require virtual for correct signing.

Notes#

Runtime capabilities are passed with ~env on operations.
S3_ENDPOINT may be a bare host or a full URI.
For streaming uploads from files, use S3.Stream.of_flow or a custom S3.Stream.source.
For streaming downloads, pass a S3.Stream.sink to S3.get_object.
Multipart upload is supported through create_multipart_upload, upload_part, complete_multipart_upload, and abort_multipart_upload.

Configure Feed