// Package s3 provides S3 client initialization and presigned URL generation // for hold services. It supports S3, Storj, and Minio storage backends. package s3 import ( "context" "fmt" "log/slog" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" ) // S3Client defines the S3 operations used by the hold service. // This interface allows mocking S3 for tests without real credentials. // Use RealS3Client to wrap *s3.Client, or MockS3Client for testing. type S3Client interface { // Multipart upload operations CreateMultipartUpload(ctx context.Context, input *awss3.CreateMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.CreateMultipartUploadOutput, error) CompleteMultipartUpload(ctx context.Context, input *awss3.CompleteMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.CompleteMultipartUploadOutput, error) AbortMultipartUpload(ctx context.Context, input *awss3.AbortMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.AbortMultipartUploadOutput, error) // Presigned URL operations - return URL string directly PresignGetObject(ctx context.Context, input *awss3.GetObjectInput, expires time.Duration) (string, error) PresignHeadObject(ctx context.Context, input *awss3.HeadObjectInput, expires time.Duration) (string, error) PresignPutObject(ctx context.Context, input *awss3.PutObjectInput, expires time.Duration) (string, error) PresignUploadPart(ctx context.Context, input *awss3.UploadPartInput, expires time.Duration) (string, error) } // RealS3Client wraps AWS SDK v2 *s3.Client to implement S3Client interface type RealS3Client struct { client *awss3.Client presign *awss3.PresignClient } // NewRealS3Client creates a new RealS3Client wrapper func NewRealS3Client(client *awss3.Client) *RealS3Client { return &RealS3Client{ client: client, presign: awss3.NewPresignClient(client), } } // CreateMultipartUpload implements S3Client func (r *RealS3Client) CreateMultipartUpload(ctx context.Context, input *awss3.CreateMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.CreateMultipartUploadOutput, error) { return r.client.CreateMultipartUpload(ctx, input, opts...) } // CompleteMultipartUpload implements S3Client func (r *RealS3Client) CompleteMultipartUpload(ctx context.Context, input *awss3.CompleteMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.CompleteMultipartUploadOutput, error) { return r.client.CompleteMultipartUpload(ctx, input, opts...) } // AbortMultipartUpload implements S3Client func (r *RealS3Client) AbortMultipartUpload(ctx context.Context, input *awss3.AbortMultipartUploadInput, opts ...func(*awss3.Options)) (*awss3.AbortMultipartUploadOutput, error) { return r.client.AbortMultipartUpload(ctx, input, opts...) } // PresignGetObject implements S3Client func (r *RealS3Client) PresignGetObject(ctx context.Context, input *awss3.GetObjectInput, expires time.Duration) (string, error) { result, err := r.presign.PresignGetObject(ctx, input, func(opts *awss3.PresignOptions) { opts.Expires = expires }) if err != nil { return "", err } return result.URL, nil } // PresignHeadObject implements S3Client func (r *RealS3Client) PresignHeadObject(ctx context.Context, input *awss3.HeadObjectInput, expires time.Duration) (string, error) { result, err := r.presign.PresignHeadObject(ctx, input, func(opts *awss3.PresignOptions) { opts.Expires = expires }) if err != nil { return "", err } return result.URL, nil } // PresignPutObject implements S3Client func (r *RealS3Client) PresignPutObject(ctx context.Context, input *awss3.PutObjectInput, expires time.Duration) (string, error) { result, err := r.presign.PresignPutObject(ctx, input, func(opts *awss3.PresignOptions) { opts.Expires = expires }) if err != nil { return "", err } return result.URL, nil } // PresignUploadPart implements S3Client func (r *RealS3Client) PresignUploadPart(ctx context.Context, input *awss3.UploadPartInput, expires time.Duration) (string, error) { result, err := r.presign.PresignUploadPart(ctx, input, func(opts *awss3.PresignOptions) { opts.Expires = expires }) if err != nil { return "", err } return result.URL, nil } // S3Service wraps an S3 client for presigned URL generation type S3Service struct { Client S3Client // S3 client for presigned URLs (interface for testability) Bucket string // S3 bucket name PathPrefix string // S3 path prefix (if any) } // NewS3Service initializes the S3 client for presigned URL generation // S3 is required - this will return an error if not properly configured func NewS3Service(params map[string]any) (*S3Service, error) { // Extract required S3 configuration region, _ := params["region"].(string) if region == "" { region = "us-east-1" // Default region } accessKey, _ := params["accesskey"].(string) secretKey, _ := params["secretkey"].(string) bucket, _ := params["bucket"].(string) if bucket == "" { return nil, fmt.Errorf("S3 bucket not configured") } // Build config options var configOpts []func(*config.LoadOptions) error configOpts = append(configOpts, config.WithRegion(region)) // Add credentials if provided (allow IAM role auth if not provided) if accessKey != "" && secretKey != "" { configOpts = append(configOpts, config.WithCredentialsProvider( credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""), )) } // Load AWS config cfg, err := config.LoadDefaultConfig(context.Background(), configOpts...) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } // Custom endpoint for S3-compatible services (Storj, MinIO, R2, etc.) endpoint, _ := params["regionendpoint"].(string) // Create S3 client client := awss3.NewFromConfig(cfg, func(o *awss3.Options) { if endpoint != "" { o.BaseEndpoint = aws.String(endpoint) o.UsePathStyle = true } }) var s3PathPrefix string // Extract path prefix if configured (rootdirectory in S3 params) if rootDir, ok := params["rootdirectory"].(string); ok && rootDir != "" { s3PathPrefix = strings.TrimPrefix(rootDir, "/") } slog.Info("S3 presigned URLs enabled", "bucket", bucket, "region", region, "pathPrefix", s3PathPrefix) // Create S3 client wrapped in RealS3Client for interface compatibility return &S3Service{ Client: NewRealS3Client(client), Bucket: bucket, PathPrefix: s3PathPrefix, }, nil } // BlobPath converts a digest (e.g., "sha256:abc123...") or temp path to a storage path // Distribution stores blobs as: /docker/registry/v2/blobs/{algorithm}/{xx}/{hash}/data // where xx is the first 2 characters of the hash for directory sharding // NOTE: Path must start with / for filesystem driver // This is used for OCI container layers (content-addressed, globally deduplicated) func BlobPath(digest string) string { // Handle temp paths (start with uploads/temp-) if strings.HasPrefix(digest, "uploads/temp-") { return fmt.Sprintf("/docker/registry/v2/%s/data", digest) } // Split digest into algorithm and hash parts := strings.SplitN(digest, ":", 2) if len(parts) != 2 { // Fallback for malformed digest return fmt.Sprintf("/docker/registry/v2/blobs/%s/data", digest) } algorithm := parts[0] hash := parts[1] // Use first 2 characters for sharding if len(hash) < 2 { return fmt.Sprintf("/docker/registry/v2/blobs/%s/%s/data", algorithm, hash) } return fmt.Sprintf("/docker/registry/v2/blobs/%s/%s/%s/data", algorithm, hash[:2], hash) }