Simple S3-like server for development purposes, written in Go
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

More robust file upload, ETag computation

+46 -9
-2
gos3dir.go
··· 45 45 46 46 srv := &server{rootDir: rootDir, logger: logger} 47 47 48 - // TODO: (Claude) "S3 returns ETags for uploaded objects. Consider adding MD5 hashing" 49 - 50 48 // TODO: Support virtual addressing style https://docs.aws.amazon.com/cli/latest/topic/s3-config.html#addressing-style 51 49 // for now it assumes path style 52 50 http.HandleFunc("GET /{$}", srv.ls)
+46 -7
server.go
··· 1 1 package main 2 2 3 3 import ( 4 + "crypto/md5" 5 + "crypto/rand" 6 + "encoding/hex" 4 7 "encoding/xml" 5 8 "errors" 6 9 "fmt" 7 10 "io" 8 11 "io/fs" 9 12 "log" 13 + "math/big" 10 14 "net/http" 11 15 "os" 12 16 "path/filepath" ··· 236 240 return 237 241 } 238 242 239 - // Safely create nested directories if needed 240 - if err := root.MkdirAll(filepath.Dir(path), 0775); err != nil { 243 + randomInt, _ := rand.Int(rand.Reader, big.NewInt(1000)) 244 + tempName := fmt.Sprintf(".tmp.%d.%s", randomInt, filepath.Base(path)) 245 + tempPath := filepath.Join(bucketName, tempName) 246 + 247 + tempFile, err := root.Create(tempPath) 248 + if err != nil { 241 249 http.Error(w, err.Error(), http.StatusInternalServerError) 242 250 return 243 251 } 252 + defer root.Remove(tempPath) 253 + defer tempFile.Close() 254 + 255 + hasher := md5.New() 256 + teeReader := io.TeeReader(r.Body, hasher) 244 257 245 - file, err := root.Create(path) 258 + bytesWritten, err := io.Copy(tempFile, teeReader) 246 259 if err != nil { 247 260 http.Error(w, err.Error(), http.StatusInternalServerError) 248 261 return 249 262 } 250 - defer file.Close() 263 + // We ignore errors here, if anything, the program will break when renaming 264 + tempFile.Sync() 265 + tempFile.Close() 266 + 267 + expectedContentLength := r.Header.Get("Content-Length") 268 + actualContentLength := strconv.FormatInt(bytesWritten, 10) 269 + if expectedContentLength != "" && expectedContentLength != actualContentLength { 270 + http.Error( 271 + w, 272 + fmt.Sprintf( 273 + "Expected Content-Length was %s, gor %s", 274 + expectedContentLength, 275 + actualContentLength, 276 + ), 277 + http.StatusInternalServerError, 278 + ) 279 + } 251 280 252 - // TODO: (Claude) "if io.Copy fails, you've already created the file. 253 - // Consider cleaning it up or using os.CreateTemp + os.Rename for atomicity." 254 - if _, err := io.Copy(file, r.Body); err != nil { 281 + // Safely create nested directories if needed 282 + // TODO: Remove dangling paths if anything fails 283 + if err := root.MkdirAll(filepath.Dir(path), 0775); err != nil { 255 284 http.Error(w, err.Error(), http.StatusInternalServerError) 256 285 return 257 286 } 287 + 288 + // Move temporary file to final destination 289 + if err := root.Rename(tempPath, path); err != nil { 290 + http.Error(w, err.Error(), http.StatusInternalServerError) 291 + return 292 + } 293 + 294 + etag := fmt.Sprintf("\"%s\"", hex.EncodeToString(hasher.Sum(nil))) 295 + w.Header().Set("ETag", etag) 296 + 258 297 w.WriteHeader(http.StatusOK) 259 298 } 260 299