+20

go.mod

reviewed

··· 3 3 go 1.26.2 4 4 5 5 require ( 6 6 + github.com/aws/aws-sdk-go-v2 v1.41.7 7 7 + github.com/aws/aws-sdk-go-v2/config v1.32.17 8 8 + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 6 9 github.com/gliderlabs/ssh v0.3.8 7 10 github.com/go-git/go-billy/v5 v5.8.0 11 11 + github.com/joho/godotenv v1.5.1 8 12 github.com/pborman/getopt/v2 v2.1.0 9 13 github.com/pmezard/go-difflib v1.0.0 10 14 github.com/spf13/pflag v1.0.10 11 15 github.com/tetratelabs/wazero v1.11.0 16 16 + go.uber.org/atomic v1.11.0 12 17 golang.org/x/term v0.41.0 13 18 golang.org/x/text v0.29.0 14 19 mvdan.cc/sh/v3 v3.13.1 ··· 16 21 17 22 require ( 18 23 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect 24 24 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect 25 25 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect 26 26 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect 27 27 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect 28 28 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect 29 29 + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect 30 30 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect 31 31 + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect 32 32 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect 33 33 + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect 34 34 + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect 35 35 + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect 36 36 + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect 37 37 + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect 38 38 + github.com/aws/smithy-go v1.25.1 // indirect 19 39 github.com/cyphar/filepath-securejoin v0.3.6 // indirect 20 40 golang.org/x/crypto v0.31.0 // indirect 21 41 golang.org/x/sys v0.42.0 // indirect

+40

go.sum

reviewed

··· 1 1 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= 2 2 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= 3 3 + github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= 4 4 + github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= 5 5 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= 6 6 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= 7 7 + github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= 8 8 + github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= 9 9 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= 10 10 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= 11 11 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= 12 12 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= 13 13 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= 14 14 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= 15 15 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= 16 16 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= 17 17 + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= 18 18 + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= 19 19 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= 20 20 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= 21 21 + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= 22 22 + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= 23 23 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= 24 24 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= 25 25 + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= 26 26 + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= 27 27 + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 h1:mxuT1xE+dI54NW3RkNjP8DUT5HXqbkiAFvfdyDFwE5c= 28 28 + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= 29 29 + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= 30 30 + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= 31 31 + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= 32 32 + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= 33 33 + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= 34 34 + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= 35 35 + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= 36 36 + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= 37 37 + github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= 38 38 + github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= 3 39 github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= 4 40 github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= 5 41 github.com/cyphar/filepath-securejoin v0.3.6 h1:4d9N5ykBnSp5Xn2JkhocYDkOpURL/18CYMpo6xB9uWM= ··· 14 50 github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= 15 51 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 16 52 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 53 53 + github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= 54 54 + github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 17 55 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 18 56 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 19 57 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= ··· 32 70 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 33 71 github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= 34 72 github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU= 73 73 + go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= 74 74 + go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 35 75 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= 36 76 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= 37 77 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=

+19

internal/s3fs/.gitignore

reviewed

··· 1 1 + # Binaries for programs and plugins 2 2 + *.exe 3 3 + *.exe~ 4 4 + *.dll 5 5 + *.so 6 6 + *.dylib 7 7 + 8 8 + # Test binary, built with `go test -c` 9 9 + *.test 10 10 + 11 11 + # Output of the go coverage tool, specifically when used with LiteIDE 12 12 + *.out 13 13 + 14 14 + # Dependency directories (remove the comment below to include it) 15 15 + # vendor/ 16 16 + 17 17 + .env 18 18 + .DS_Store 19 19 + tmp/

+21

internal/s3fs/LICENSE

reviewed

··· 1 1 + MIT License 2 2 + 3 3 + Copyright (c) 2022 Austin Poor 4 4 + 5 5 + Permission is hereby granted, free of charge, to any person obtaining a copy 6 6 + of this software and associated documentation files (the "Software"), to deal 7 7 + in the Software without restriction, including without limitation the rights 8 8 + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 9 + copies of the Software, and to permit persons to whom the Software is 10 10 + furnished to do so, subject to the following conditions: 11 11 + 12 12 + The above copyright notice and this permission notice shall be included in all 13 13 + copies or substantial portions of the Software. 14 14 + 15 15 + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 16 + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 17 + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 18 + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 19 + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 20 + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 21 + SOFTWARE.

+5

internal/s3fs/README.md

reviewed

··· 1 1 + # s3fs 2 2 + 3 3 + _created by Austin Poor_ 4 4 + 5 5 + An implementation of the [go-billy `Filesystem`](https://pkg.go.dev/github.com/go-git/go-billy/v5#Filesystem) for treating an S3 bucket like a filesystem.

+140

internal/s3fs/basic.go

reviewed

··· 1 1 + // basic.go implements the interface billy.Basic 2 2 + 3 3 + package main 4 4 + 5 5 + import ( 6 6 + "context" 7 7 + "errors" 8 8 + "fmt" 9 9 + "os" 10 10 + "path" 11 11 + 12 12 + "github.com/aws/aws-sdk-go-v2/service/s3" 13 13 + "github.com/go-git/go-billy/v5" 14 14 + ) 15 15 + 16 16 + const ( 17 17 + O_RDONLY int = os.O_RDONLY // open the file read-only. 18 18 + O_WRONLY int = os.O_WRONLY // open the file write-only. 19 19 + O_WRMULTIPART int = 0x4 // open the file for write-only using multipart upload. 20 20 + 21 21 + SupportedOFlags = O_RDONLY | O_WRONLY | O_WRMULTIPART // supported open flags for s3fs 22 22 + ) 23 23 + 24 24 + var ( 25 25 + ErrOpenFlagNotSupported = errors.New("open flag not supported") 26 26 + ) 27 27 + 28 28 + // Create implements billy.Basic 29 29 + // Create creates the named file with mode 0666 (before umask), truncating 30 30 + // it if it already exists. If successful, methods on the returned File can 31 31 + // be used for I/O; the associated file descriptor has mode O_RDWR. 32 32 + func (fs3 *S3FS) Create(filename string) (billy.File, error) { 33 33 + return fs3.OpenFile(filename, O_WRONLY, 0666) 34 34 + } 35 35 + 36 36 + // Open opens the named file for reading. If successful, methods on the 37 37 + // returned file can be used for reading; the associated file descriptor has 38 38 + // mode O_RDONLY. 39 39 + func (fs3 *S3FS) Open(filename string) (billy.File, error) { 40 40 + return fs3.OpenFile(filename, O_RDONLY, 0666) 41 41 + } 42 42 + 43 43 + // OpenFile is the generalized open call; most users will use Open or Create 44 44 + // instead. It opens the named file with specified flag (O_RDONLY etc.) and 45 45 + // perm, (0666 etc.) if applicable. If successful, methods on the returned 46 46 + // File can be used for I/O. 47 47 + func (fs3 *S3FS) OpenFile(filename string, flag int, perm os.FileMode) (billy.File, error) { 48 48 + // Is the supplied flag supported? 49 49 + if flag&SupportedOFlags != flag { 50 50 + return nil, errors.New("unsupported open flag") 51 51 + } 52 52 + 53 53 + // Get the file path 54 54 + p := path.Join(fs3.root, filename) 55 55 + 56 56 + switch flag & SupportedOFlags { 57 57 + case O_RDONLY: 58 58 + return newS3ReadFile(fs3.client, fs3.bucket, p) 59 59 + 60 60 + case O_WRONLY: 61 61 + return newS3WriteFile(fs3.client, fs3.bucket, p) 62 62 + 63 63 + case O_WRMULTIPART: 64 64 + return newS3MultipartUploadFile(fs3.client, fs3.bucket, p) 65 65 + 66 66 + default: 67 67 + return nil, errors.New("unsupported open flag") 68 68 + } 69 69 + } 70 70 + 71 71 + // Stat returns a FileInfo describing the named file. 72 72 + func (fs3 *S3FS) Stat(filename string) (os.FileInfo, error) { 73 73 + return nil, errors.New("not implemented") 74 74 + } 75 75 + 76 76 + // Rename renames (moves) oldpath to newpath. If newpath already exists and 77 77 + // is not a directory, Rename replaces it. OS-specific restrictions may 78 78 + // apply when oldpath and newpath are in different directories. 79 79 + func (fs3 *S3FS) Rename(oldpath, newpath string) error { 80 80 + // TODO: Validate the paths? 81 81 + 82 82 + // Create a context 83 83 + ctx := context.TODO() // TODO: Get user-supplied context? 84 84 + 85 85 + // Format the paths 86 86 + src := path.Join(fs3.root, oldpath) 87 87 + dst := path.Join(fs3.root, newpath) 88 88 + 89 89 + // Send the copy request 90 90 + _, err := fs3.client.CopyObject(ctx, &s3.CopyObjectInput{ 91 91 + Bucket: &fs3.bucket, 92 92 + CopySource: &src, 93 93 + Key: &dst, 94 94 + }) 95 95 + if err != nil { 96 96 + return fmt.Errorf("failed to rename file: %s", err) 97 97 + } 98 98 + 99 99 + // Delete the old file 100 100 + // TODO: Parse the response? 101 101 + _, err = fs3.client.DeleteObject(ctx, &s3.DeleteObjectInput{ 102 102 + Bucket: &fs3.bucket, 103 103 + Key: &src, 104 104 + }) 105 105 + if err != nil { 106 106 + return fmt.Errorf("failed to remove file: %s", err) 107 107 + } 108 108 + 109 109 + return nil 110 110 + } 111 111 + 112 112 + // Remove removes the named file or directory. 113 113 + func (fs3 *S3FS) Remove(filename string) error { 114 114 + // TODO: Validate the path? 115 115 + // ... 116 116 + 117 117 + // Create a context 118 118 + ctx := context.TODO() // TODO: Get user-supplied context? 119 119 + 120 120 + // Format the path 121 121 + p := path.Join(fs3.root, filename) 122 122 + 123 123 + // Send the request 124 124 + // TODO: Parse the response? 125 125 + _, err := fs3.client.DeleteObject(ctx, &s3.DeleteObjectInput{ 126 126 + Bucket: &fs3.bucket, 127 127 + Key: &p, 128 128 + }) 129 129 + if err != nil { 130 130 + return fmt.Errorf("failed to remove file: %s", err) 131 131 + } 132 132 + return nil 133 133 + } 134 134 + 135 135 + // Join joins any number of path elements into a single path 136 136 + func (fs3 *S3FS) Join(elem ...string) string { 137 137 + j := path.Join(elem...) 138 138 + c := path.Clean(j) 139 139 + return c 140 140 + }

+29

internal/s3fs/chroot.go

reviewed

··· 1 1 + // chroot.go implements the interface billy.Chroot 2 2 + 3 3 + package main 4 4 + 5 5 + import "github.com/go-git/go-billy/v5" 6 6 + 7 7 + // Chroot returns a new filesystem from the same type where the new root is 8 8 + // the given path. Files outside of the designated directory tree cannot be 9 9 + // accessed. 10 10 + func (fs3 *S3FS) Chroot(path string) (billy.Filesystem, error) { 11 11 + // TODO: Check that path is a valid subdirectory of the current root 12 12 + // ... 13 13 + 14 14 + // Calculate the new root 15 15 + p := fs3.Join(fs3.root, path) 16 16 + 17 17 + // Create the new S3FS with the new root directory 18 18 + nfs := &S3FS{ 19 19 + client: fs3.client, 20 20 + bucket: fs3.bucket, 21 21 + root: p, 22 22 + } 23 23 + return nfs, nil 24 24 + } 25 25 + 26 26 + // Root returns the root path of the filesystem. 27 27 + func (fs3 *S3FS) Root() string { 28 28 + return fs3.root 29 29 + }

+75

internal/s3fs/dir.go

reviewed

··· 1 1 + // dir.go implements the interface billy.Dir 2 2 + 3 3 + package main 4 4 + 5 5 + import ( 6 6 + "context" 7 7 + "errors" 8 8 + "os" 9 9 + 10 10 + "github.com/aws/aws-sdk-go-v2/aws" 11 11 + "github.com/aws/aws-sdk-go-v2/service/s3" 12 12 + ) 13 13 + 14 14 + // ReadDir reads the directory named by dirname and returns a list of 15 15 + // directory entries sorted by filename. 16 16 + func (fs3 *S3FS) ReadDir(path string) ([]os.FileInfo, error) { 17 17 + // p := fs3.cleanPath(fs3.root, path) 18 18 + // if p != "" { 19 19 + // p += "/" 20 20 + // } 21 21 + // fmt.Println("ReadDir:", p) 22 22 + p := path 23 23 + 24 24 + // Create a context with a timeout 25 25 + ctx := context.TODO() // TODO: Get user context? 26 26 + 27 27 + var ct *string 28 28 + var dirs []os.FileInfo 29 29 + var files []os.FileInfo 30 30 + for { 31 31 + res, err := fs3.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ 32 32 + Bucket: &fs3.bucket, 33 33 + Prefix: &p, 34 34 + ContinuationToken: ct, 35 35 + Delimiter: &fs3.separator, 36 36 + }) 37 37 + if err != nil { 38 38 + return nil, err 39 39 + } 40 40 + 41 41 + // Add the directories to the list 42 42 + for _, d := range res.CommonPrefixes { 43 43 + dirs = append(dirs, newDirInfo(*d.Prefix)) 44 44 + } 45 45 + 46 46 + // Add the files to the list 47 47 + for _, f := range res.Contents { 48 48 + files = append(files, newFileInfo( 49 49 + aws.ToString(f.Key), 50 50 + *f.Size, 51 51 + aws.ToTime(f.LastModified), 52 52 + )) 53 53 + } 54 54 + 55 55 + // Set the last key 56 56 + ct = res.NextContinuationToken 57 57 + 58 58 + // If there are no more keys, break 59 59 + if !*res.IsTruncated { 60 60 + break 61 61 + } 62 62 + } 63 63 + 64 64 + // Join the directories and files & return 65 65 + res := append(dirs, files...) 66 66 + return res, nil 67 67 + } 68 68 + 69 69 + // MkdirAll creates a directory named path, along with any necessary 70 70 + // parents, and returns nil, or else returns an error. The permission bits 71 71 + // perm are used for all directories that MkdirAll creates. If path is/ 72 72 + // already a directory, MkdirAll does nothing and returns nil. 73 73 + func (fs3 *S3FS) MkdirAll(filename string, perm os.FileMode) error { 74 74 + return errors.New("not implemented") 75 75 + }

+355

internal/s3fs/file.go

reviewed

··· 1 1 + package main 2 2 + 3 3 + import ( 4 4 + "bytes" 5 5 + "context" 6 6 + "errors" 7 7 + "fmt" 8 8 + "io/fs" 9 9 + "io/ioutil" 10 10 + "os" 11 11 + 12 12 + "github.com/aws/aws-sdk-go-v2/service/s3" 13 13 + "go.uber.org/atomic" 14 14 + ) 15 15 + 16 16 + const ( 17 17 + ModeMultipartUpload os.FileMode = fs.ModePerm + 1 // Custom os.FileMode for S3 multipart upload 18 18 + ) 19 19 + 20 20 + var ( 21 21 + ErrLockNotSupported = errors.New("lock not supported by s3") 22 22 + ErrTruncateNotSupported = errors.New("truncate not supported by s3") 23 23 + ErrFileClosed = errors.New("file is closed") 24 24 + ErrCantWriteToReadOnly = errors.New("can't write to read-only file") 25 25 + ErrCantReadFromWriteOnly = errors.New("can't read from write-only file") 26 26 + ) 27 27 + 28 28 + // s3ReadFile implements billy.File for S3, and represents a file opened in read mode. 29 29 + // 30 30 + // Upon creation, the file is loaded from S3. 31 31 + type s3ReadFile struct { 32 32 + client *s3.Client // s3 skd client 33 33 + bucket string // S3 bucket name 34 34 + key string // File object's key in S3 35 35 + closed bool // Is the file closed? 36 36 + reader *bytes.Reader // Buffer for file contents 37 37 + } 38 38 + 39 39 + // newS3ReadFile creates a new s3ReadFile. 40 40 + func newS3ReadFile(client *s3.Client, bucket, key string) (*s3ReadFile, error) { 41 41 + // TODO: Check if the file exists 42 42 + // ... 43 43 + 44 44 + // Create the context 45 45 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 46 46 + 47 47 + // Run the GetObject operation 48 48 + res, err := client.GetObject(ctx, &s3.GetObjectInput{ 49 49 + Bucket: &bucket, 50 50 + Key: &key, 51 51 + }) 52 52 + if err != nil { 53 53 + return nil, fmt.Errorf("unable to perform GetObject operation: %w", err) 54 54 + } 55 55 + 56 56 + // Read the file contents and store in a bytes reader 57 57 + buf, err := ioutil.ReadAll(res.Body) 58 58 + if err != nil { 59 59 + return nil, fmt.Errorf("unable to read file body: %w", err) 60 60 + } 61 61 + reader := bytes.NewReader(buf) 62 62 + 63 63 + // Return the file 64 64 + return &s3ReadFile{ 65 65 + client: client, 66 66 + bucket: bucket, 67 67 + key: key, 68 68 + reader: reader, 69 69 + }, nil 70 70 + } 71 71 + 72 72 + // Name returns the name of the file as presented to Open. 73 73 + func (f *s3ReadFile) Name() string { 74 74 + return f.key 75 75 + } 76 76 + 77 77 + // Write implements os.Writer for billy.File 78 78 + func (f *s3ReadFile) Write(p []byte) (n int, err error) { 79 79 + return 0, ErrCantWriteToReadOnly 80 80 + } 81 81 + 82 82 + // Read implements os.Reader for billy.File 83 83 + func (f *s3ReadFile) Read(p []byte) (n int, err error) { 84 84 + return f.reader.Read(p) 85 85 + } 86 86 + 87 87 + // ReadAt implements io.ReaderAt for billy.File 88 88 + func (f *s3ReadFile) ReadAt(p []byte, off int64) (n int, err error) { 89 89 + return f.reader.ReadAt(p, off) 90 90 + } 91 91 + 92 92 + // Seek implements io.Seeker for billy.File 93 93 + func (f *s3ReadFile) Seek(offset int64, whence int) (int64, error) { 94 94 + return f.reader.Seek(offset, whence) 95 95 + } 96 96 + 97 97 + // Close implements io.Closer for billy.File 98 98 + func (f *s3ReadFile) Close() error { 99 99 + // Was the file already closed? 100 100 + if f.closed { 101 101 + return ErrFileClosed 102 102 + } 103 103 + 104 104 + // Close the underlying file 105 105 + f.reader = nil 106 106 + 107 107 + // Mark the file as closed 108 108 + f.closed = true 109 109 + 110 110 + return nil 111 111 + } 112 112 + 113 113 + // Lock locks the file like e.g. flock. It protects against access from 114 114 + // other processes. 115 115 + func (f *s3ReadFile) Lock() error { 116 116 + return ErrLockNotSupported 117 117 + } 118 118 + 119 119 + // Unlock unlocks the file. 120 120 + func (f *s3ReadFile) Unlock() error { 121 121 + return ErrLockNotSupported 122 122 + } 123 123 + 124 124 + // Truncate the file. 125 125 + func (f *s3ReadFile) Truncate(size int64) error { 126 126 + return ErrTruncateNotSupported 127 127 + } 128 128 + 129 129 + // s3WriteFile stores a file opened in write mode and implements billy.File 130 130 + // 131 131 + // Upon creation, a buffer is created to store the file contents. Upon close, 132 132 + // the file is uploaded to S3. 133 133 + type s3WriteFile struct { 134 134 + client *s3.Client // s3 skd client 135 135 + bucket string // S3 bucket name 136 136 + key string // File object's key in S3 137 137 + closed bool // Is the file closed? 138 138 + buf *bytes.Buffer // Buffer for storing the file before it's uploaded 139 139 + } 140 140 + 141 141 + // newS3WriteFile creates a new s3ReadFile. 142 142 + func newS3WriteFile(client *s3.Client, bucket, key string) (*s3WriteFile, error) { 143 143 + // TODO: Validate the key 144 144 + // ... 145 145 + 146 146 + return &s3WriteFile{ 147 147 + client: client, 148 148 + bucket: bucket, 149 149 + key: key, 150 150 + buf: bytes.NewBuffer(nil), 151 151 + }, nil 152 152 + } 153 153 + 154 154 + // Name returns the name of the file as presented to Open. 155 155 + func (f *s3WriteFile) Name() string { 156 156 + return f.key 157 157 + } 158 158 + 159 159 + // Write implements os.Writer for billy.File 160 160 + func (f *s3WriteFile) Write(p []byte) (n int, err error) { 161 161 + return 0, nil 162 162 + } 163 163 + 164 164 + // Read implements os.Reader for billy.File 165 165 + func (f *s3WriteFile) Read(p []byte) (n int, err error) { 166 166 + return 0, ErrCantReadFromWriteOnly 167 167 + } 168 168 + 169 169 + // ReadAt implements io.ReaderAt for billy.File 170 170 + func (f *s3WriteFile) ReadAt(p []byte, off int64) (n int, err error) { 171 171 + return 0, ErrCantReadFromWriteOnly 172 172 + } 173 173 + 174 174 + // Seek implements io.Seeker for billy.File 175 175 + func (f *s3WriteFile) Seek(offset int64, whence int) (int64, error) { 176 176 + return 0, errors.New("not implemented") 177 177 + } 178 178 + 179 179 + // Close implements io.Closer for billy.File 180 180 + func (f *s3WriteFile) Close() error { 181 181 + if f.closed { 182 182 + return ErrFileClosed 183 183 + } 184 184 + 185 185 + // Set to closed 186 186 + f.closed = true 187 187 + 188 188 + // Extract the body from the buffer 189 189 + body := bytes.NewReader(f.buf.Bytes()) 190 190 + 191 191 + // Create the context 192 192 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 193 193 + 194 194 + // Run the GetObject operation 195 195 + // TODO: Currently `res` is not used. Should it be? 196 196 + _, err := f.client.PutObject(ctx, &s3.PutObjectInput{ 197 197 + Bucket: &f.bucket, 198 198 + Key: &f.key, 199 199 + Body: body, 200 200 + }) 201 201 + if err != nil { 202 202 + return fmt.Errorf("unable to perform GetObject operation: %w", err) 203 203 + } 204 204 + 205 205 + return nil 206 206 + } 207 207 + 208 208 + // Lock locks the file like e.g. flock. It protects against access from 209 209 + // other processes. 210 210 + func (f *s3WriteFile) Lock() error { 211 211 + return ErrLockNotSupported 212 212 + } 213 213 + 214 214 + // Unlock unlocks the file. 215 215 + func (f *s3WriteFile) Unlock() error { 216 216 + return ErrLockNotSupported 217 217 + } 218 218 + 219 219 + // Truncate the file. 220 220 + func (f *s3WriteFile) Truncate(size int64) error { 221 221 + return ErrTruncateNotSupported 222 222 + } 223 223 + 224 224 + // s3MultipartUploadFile implements billy.File 225 225 + type s3MultipartUploadFile struct { 226 226 + client *s3.Client // s3 skd client 227 227 + bucket string // S3 bucket name 228 228 + key string // File object's key in S3 229 229 + closed bool // Is the file closed? 230 230 + uploadID string // S3 multipart upload ID 231 231 + uploadN *atomic.Int32 // Counter tracking the number of uploads 232 232 + } 233 233 + 234 234 + // newS3MultipartUploadFile creates a new s3ReadFile. 235 235 + func newS3MultipartUploadFile(client *s3.Client, bucket, key string) (*s3MultipartUploadFile, error) { 236 236 + // TODO: Check if the file exists 237 237 + // ... 238 238 + 239 239 + // Create the context 240 240 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 241 241 + 242 242 + // Run the GetObject operation 243 243 + res, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ 244 244 + Bucket: &bucket, 245 245 + Key: &key, 246 246 + }) 247 247 + if err != nil { 248 248 + return nil, fmt.Errorf("unable to create multipart upload: %w", err) 249 249 + } 250 250 + 251 251 + // Return the file 252 252 + return &s3MultipartUploadFile{ 253 253 + client: client, 254 254 + bucket: bucket, 255 255 + key: key, 256 256 + uploadID: *res.UploadId, 257 257 + uploadN: atomic.NewInt32(1), 258 258 + }, nil 259 259 + } 260 260 + 261 261 + // Name returns the name of the file as presented to Open. 262 262 + func (f *s3MultipartUploadFile) Name() string { 263 263 + return f.key 264 264 + } 265 265 + 266 266 + // Write implements os.Writer for billy.File 267 267 + func (f *s3MultipartUploadFile) Write(p []byte) (n int, err error) { 268 268 + // Get the size of the data being written 269 269 + n = len(p) 270 270 + 271 271 + // Create a context for the operation 272 272 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 273 273 + 274 274 + // Create a reader for the data 275 275 + r := bytes.NewReader(p) 276 276 + 277 277 + // Get the part number 278 278 + pn := f.uploadN.Load() 279 279 + 280 280 + // Run the UploadPart operation 281 281 + _, err = f.client.UploadPart(ctx, &s3.UploadPartInput{ 282 282 + Bucket: &f.bucket, 283 283 + Key: &f.key, 284 284 + UploadId: &f.uploadID, 285 285 + PartNumber: new(pn), 286 286 + Body: r, 287 287 + }) 288 288 + if err != nil { 289 289 + return 0, fmt.Errorf("unable to upload part %d: %w", pn, err) 290 290 + } 291 291 + 292 292 + // Increment the part number 293 293 + f.uploadN.Add(1) 294 294 + 295 295 + // Return the number of bytes written 296 296 + return n, nil 297 297 + } 298 298 + 299 299 + // Read implements os.Reader for billy.File 300 300 + func (f *s3MultipartUploadFile) Read(p []byte) (n int, err error) { 301 301 + return 0, ErrCantReadFromWriteOnly 302 302 + } 303 303 + 304 304 + // ReadAt implements io.ReaderAt for billy.File 305 305 + func (f *s3MultipartUploadFile) ReadAt(p []byte, off int64) (n int, err error) { 306 306 + return 0, ErrCantReadFromWriteOnly 307 307 + } 308 308 + 309 309 + // Seek implements io.Seeker for billy.File 310 310 + func (f *s3MultipartUploadFile) Seek(offset int64, whence int) (int64, error) { 311 311 + return 0, errors.New("seek not implemented") 312 312 + } 313 313 + 314 314 + // Close implements io.Closer for billy.File 315 315 + func (f *s3MultipartUploadFile) Close() error { 316 316 + // Check if the file has been closed 317 317 + if f.closed { 318 318 + return ErrFileClosed 319 319 + } 320 320 + 321 321 + // Set to closed 322 322 + f.closed = true 323 323 + 324 324 + // Create the context 325 325 + ctx := context.TODO() // TODO: How can user-supplied contexts be supported? 326 326 + 327 327 + // Complete the multipart upload 328 328 + // TODO: Currently `res` is not used. Should it be? 329 329 + _, err := f.client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ 330 330 + Bucket: &f.bucket, 331 331 + Key: &f.key, 332 332 + UploadId: &f.uploadID, 333 333 + }) 334 334 + if err != nil { 335 335 + return fmt.Errorf("unable to complete multipart upload: %w", err) 336 336 + } 337 337 + 338 338 + return nil 339 339 + } 340 340 + 341 341 + // Lock locks the file like e.g. flock. It protects against access from 342 342 + // other processes. 343 343 + func (f *s3MultipartUploadFile) Lock() error { 344 344 + return ErrLockNotSupported 345 345 + } 346 346 + 347 347 + // Unlock unlocks the file. 348 348 + func (f *s3MultipartUploadFile) Unlock() error { 349 349 + return ErrLockNotSupported 350 350 + } 351 351 + 352 352 + // Truncate the file. 353 353 + func (f *s3MultipartUploadFile) Truncate(size int64) error { 354 354 + return ErrTruncateNotSupported 355 355 + }

+56

internal/s3fs/fileinfo.go

reviewed

··· 1 1 + package main 2 2 + 3 3 + import ( 4 4 + "io/fs" 5 5 + "os" 6 6 + "time" 7 7 + ) 8 8 + 9 9 + // s3FileInfo implements os.FileInfo 10 10 + type s3FileInfo struct { 11 11 + name string 12 12 + size int64 13 13 + mode os.FileMode 14 14 + modTime time.Time 15 15 + } 16 16 + 17 17 + func newFileInfo(name string, size int64, modTime time.Time) os.FileInfo { 18 18 + return s3FileInfo{ 19 19 + name: name, 20 20 + size: size, 21 21 + mode: 0666, 22 22 + modTime: modTime, 23 23 + } 24 24 + } 25 25 + 26 26 + func newDirInfo(name string) os.FileInfo { 27 27 + return s3FileInfo{ 28 28 + name: name, 29 29 + mode: fs.ModeDir, 30 30 + modTime: time.Time{}, 31 31 + } 32 32 + } 33 33 + 34 34 + func (fi s3FileInfo) Name() string { 35 35 + return fi.name 36 36 + } 37 37 + 38 38 + func (fi s3FileInfo) Size() int64 { 39 39 + return fi.size 40 40 + } 41 41 + 42 42 + func (fi s3FileInfo) Mode() os.FileMode { 43 43 + return fi.mode 44 44 + } 45 45 + 46 46 + func (fi s3FileInfo) IsDir() bool { 47 47 + return fi.mode.IsDir() 48 48 + } 49 49 + 50 50 + func (fi s3FileInfo) Sys() interface{} { 51 51 + return nil 52 52 + } 53 53 + 54 54 + func (fi s3FileInfo) ModTime() time.Time { 55 55 + return fi.modTime 56 56 + }

+53

internal/s3fs/filesystem.go

reviewed

··· 1 1 + package main 2 2 + 3 3 + import ( 4 4 + "fmt" 5 5 + "path" 6 6 + 7 7 + "github.com/aws/aws-sdk-go-v2/service/s3" 8 8 + "github.com/go-git/go-billy/v5" 9 9 + ) 10 10 + 11 11 + const ( 12 12 + DefaultSeparator = "/" 13 13 + ) 14 14 + 15 15 + type S3FS struct { 16 16 + client *s3.Client 17 17 + bucket string 18 18 + root string 19 19 + separator string 20 20 + } 21 21 + 22 22 + // NewS3FS creates a new S3FS Filesystem. 23 23 + func NewS3FS(client *s3.Client, bucket string) (billy.Filesystem, error) { 24 24 + // Check for a non-nil client 25 25 + if client == nil { 26 26 + return nil, fmt.Errorf("s3 client cannot be nil") 27 27 + } 28 28 + return &S3FS{ 29 29 + client: client, 30 30 + bucket: bucket, 31 31 + root: "", 32 32 + separator: DefaultSeparator, 33 33 + }, nil 34 34 + } 35 35 + 36 36 + // Capabilities returns the filesystem capabilities. 37 37 + func (fs3 *S3FS) Capabilities() billy.Capability { 38 38 + return billy.ReadCapability | billy.WriteCapability 39 39 + } 40 40 + 41 41 + func (fs3 *S3FS) cleanPath(p ...string) string { 42 42 + // Join the path elements 43 43 + j := path.Join(p...) 44 44 + 45 45 + // Clean the path before joining to root 46 46 + c := path.Clean(j) 47 47 + 48 48 + // Join the root and cleaned path 49 49 + f := path.Join(fs3.root, c) 50 50 + 51 51 + // Return the full path 52 52 + return path.Clean(f) 53 53 + }

+48

internal/s3fs/main.go

reviewed

··· 1 1 + package main 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "fmt" 6 6 + "os" 7 7 + 8 8 + "github.com/aws/aws-sdk-go-v2/config" 9 9 + "github.com/aws/aws-sdk-go-v2/service/s3" 10 10 + "github.com/joho/godotenv" 11 11 + ) 12 12 + 13 13 + var BucketName string 14 14 + 15 15 + func init() { 16 16 + err := godotenv.Load() 17 17 + if err != nil { 18 18 + panic(err) 19 19 + } 20 20 + BucketName = os.Getenv("BUCKET_NAME") 21 21 + } 22 22 + 23 23 + func main() { 24 24 + // fmt.Println(BucketName) 25 25 + 26 26 + cfg, err := config.LoadDefaultConfig(context.Background()) 27 27 + if err != nil { 28 28 + panic(err) 29 29 + } 30 30 + client := s3.NewFromConfig(cfg) 31 31 + 32 32 + s3fs, err := NewS3FS(client, BucketName) 33 33 + if err != nil { 34 34 + panic(err) 35 35 + } 36 36 + fmt.Printf("s3fs.Root() = %q\n", s3fs.Root()) 37 37 + fmt.Println(s3fs.Join(s3fs.Root(), "hello/", "/")) 38 38 + 39 39 + files, err := s3fs.ReadDir("foo/") 40 40 + if err != nil { 41 41 + panic(err) 42 42 + } 43 43 + fmt.Printf("Found %d files\n", len(files)) 44 44 + for _, file := range files { 45 45 + fmt.Println(file.Name()) 46 46 + } 47 47 + 48 48 + }

+39

internal/s3fs/symlink.go

reviewed

··· 1 1 + // symlink.go implements the interface billy.Symlink 2 2 + 3 3 + package main 4 4 + 5 5 + import ( 6 6 + "errors" 7 7 + "os" 8 8 + ) 9 9 + 10 10 + var ( 11 11 + ErrSymLinkNotSupported = errors.New("symlink not supported by s3") 12 12 + ) 13 13 + 14 14 + // Lstat returns a FileInfo describing the named file. If the file is a 15 15 + // symbolic link, the returned FileInfo describes the symbolic link. Lstat 16 16 + // makes no attempt to follow the link. 17 17 + // 18 18 + // NOTE: Lstat is not supported by s3. It always returns an error. 19 19 + // (This may be revised in the future.) 20 20 + func (fs3 *S3FS) Lstat(filename string) (os.FileInfo, error) { 21 21 + return nil, ErrSymLinkNotSupported 22 22 + } 23 23 + 24 24 + // Symlink creates a symbolic-link from link to target. target may be an 25 25 + // absolute or relative path, and need not refer to an existing node. 26 26 + // Parent directories of link are created as necessary. 27 27 + // 28 28 + // NOTE: Symlink is not supported by s3. It always returns an error. 29 29 + func (fs3 *S3FS) Symlink(target, link string) error { 30 30 + return ErrSymLinkNotSupported 31 31 + } 32 32 + 33 33 + // Readlink returns the target path of link. 34 34 + // 35 35 + // NOTE: Readlink is not supported by s3. It always returns an error. 36 36 + // (This may be revised in the future.) 37 37 + func (fs3 *S3FS) Readlink(link string) (string, error) { 38 38 + return "", ErrSymLinkNotSupported 39 39 + }

+17

internal/s3fs/tempfile.go

reviewed

··· 1 1 + // tempfile.go implements the interface billy.TempFile 2 2 + 3 3 + package main 4 4 + 5 5 + import "github.com/go-git/go-billy/v5" 6 6 + 7 7 + // TempFile creates a new temporary file in the directory dir with a name 8 8 + // beginning with prefix, opens the file for reading and writing, and 9 9 + // returns the resulting *os.File. If dir is the empty string, TempFile 10 10 + // uses the default directory for temporary files (see os.TempDir). 11 11 + // Multiple programs calling TempFile simultaneously will not choose the 12 12 + // same file. The caller can use f.Name() to find the pathname of the file. 13 13 + // It is the caller's responsibility to remove the file when no longer 14 14 + // needed. 15 15 + func (fs3 *S3FS) TempFile(dir, prefix string) (billy.File, error) { 16 16 + return nil, nil 17 17 + }