Simple S3-like server for development purposes, written in Go
More robust file upload, ETag computation
+46
-9
-2
gos3dir.go
-2
gos3dir.go
···
45
45
46
46
srv := &server{rootDir: rootDir, logger: logger}
47
47
48
-
// TODO: (Claude) "S3 returns ETags for uploaded objects. Consider adding MD5 hashing"
49
-
50
48
// TODO: Support virtual addressing style https://docs.aws.amazon.com/cli/latest/topic/s3-config.html#addressing-style
51
49
// for now it assumes path style
52
50
http.HandleFunc("GET /{$}", srv.ls)
+46
-7
server.go
+46
-7
server.go
···
1
1
package main
2
2
3
3
import (
4
+
"crypto/md5"
5
+
"crypto/rand"
6
+
"encoding/hex"
4
7
"encoding/xml"
5
8
"errors"
6
9
"fmt"
7
10
"io"
8
11
"io/fs"
9
12
"log"
13
+
"math/big"
10
14
"net/http"
11
15
"os"
12
16
"path/filepath"
···
236
240
return
237
241
}
238
242
239
-
// Safely create nested directories if needed
240
-
if err := root.MkdirAll(filepath.Dir(path), 0775); err != nil {
243
+
randomInt, _ := rand.Int(rand.Reader, big.NewInt(1000))
244
+
tempName := fmt.Sprintf(".tmp.%d.%s", randomInt, filepath.Base(path))
245
+
tempPath := filepath.Join(bucketName, tempName)
246
+
247
+
tempFile, err := root.Create(tempPath)
248
+
if err != nil {
241
249
http.Error(w, err.Error(), http.StatusInternalServerError)
242
250
return
243
251
}
252
+
defer root.Remove(tempPath)
253
+
defer tempFile.Close()
254
+
255
+
hasher := md5.New()
256
+
teeReader := io.TeeReader(r.Body, hasher)
244
257
245
-
file, err := root.Create(path)
258
+
bytesWritten, err := io.Copy(tempFile, teeReader)
246
259
if err != nil {
247
260
http.Error(w, err.Error(), http.StatusInternalServerError)
248
261
return
249
262
}
250
-
defer file.Close()
263
+
// We ignore errors here, if anything, the program will break when renaming
264
+
tempFile.Sync()
265
+
tempFile.Close()
266
+
267
+
expectedContentLength := r.Header.Get("Content-Length")
268
+
actualContentLength := strconv.FormatInt(bytesWritten, 10)
269
+
if expectedContentLength != "" && expectedContentLength != actualContentLength {
270
+
http.Error(
271
+
w,
272
+
fmt.Sprintf(
273
+
"Expected Content-Length was %s, gor %s",
274
+
expectedContentLength,
275
+
actualContentLength,
276
+
),
277
+
http.StatusInternalServerError,
278
+
)
279
+
}
251
280
252
-
// TODO: (Claude) "if io.Copy fails, you've already created the file.
253
-
// Consider cleaning it up or using os.CreateTemp + os.Rename for atomicity."
254
-
if _, err := io.Copy(file, r.Body); err != nil {
281
+
// Safely create nested directories if needed
282
+
// TODO: Remove dangling paths if anything fails
283
+
if err := root.MkdirAll(filepath.Dir(path), 0775); err != nil {
255
284
http.Error(w, err.Error(), http.StatusInternalServerError)
256
285
return
257
286
}
287
+
288
+
// Move temporary file to final destination
289
+
if err := root.Rename(tempPath, path); err != nil {
290
+
http.Error(w, err.Error(), http.StatusInternalServerError)
291
+
return
292
+
}
293
+
294
+
etag := fmt.Sprintf("\"%s\"", hex.EncodeToString(hasher.Sum(nil)))
295
+
w.Header().Set("ETag", etag)
296
+
258
297
w.WriteHeader(http.StatusOK)
259
298
}
260
299