loading up the forgejo repo on tangled to test page performance
Make minio package support legacy MD5 checksum (#23768)
A feedback from discord:
https://discord.com/channels/322538954119184384/561007778139734027/1090185427115319386
Some storages like:
* https://developers.cloudflare.com/r2/api/s3/api/
* https://www.backblaze.com/b2/docs/s3_compatible_api.html
They do not support "x-amz-checksum-algorithm" header
But minio recently uses that header with CRC32C by default. So we have
to tell minio to use legacy MD5 checksum.
I guess this needs to be backported because IIRC we 1.19 and 1.20 are
using similar minio package.
The minio package code for SendContentMD5 looks like this:
<details>
<img width="755" alt="image"
src="https://user-images.githubusercontent.com/2114189/228186768-4f2f6f67-62b9-4aee-9251-5af714ad9674.png">
</details>
authored by
wxiaoguang
and committed by
GitHub
5727056e
6a0ef719
+75
-25
+19
-8
cmd/migrate_storage.go
+19
-8
cmd/migrate_storage.go
···
72
72
cli.StringFlag{
73
73
Name: "minio-base-path",
74
74
Value: "",
75
-
Usage: "Minio storage basepath on the bucket",
75
+
Usage: "Minio storage base path on the bucket",
76
76
},
77
77
cli.BoolFlag{
78
78
Name: "minio-use-ssl",
79
79
Usage: "Enable SSL for minio",
80
+
},
81
+
cli.BoolFlag{
82
+
Name: "minio-insecure-skip-verify",
83
+
Usage: "Skip SSL verification",
84
+
},
85
+
cli.StringFlag{
86
+
Name: "minio-checksum-algorithm",
87
+
Value: "",
88
+
Usage: "Minio checksum algorithm (default/md5)",
80
89
},
81
90
},
82
91
}
···
168
177
dstStorage, err = storage.NewMinioStorage(
169
178
stdCtx,
170
179
storage.MinioStorageConfig{
171
-
Endpoint: ctx.String("minio-endpoint"),
172
-
AccessKeyID: ctx.String("minio-access-key-id"),
173
-
SecretAccessKey: ctx.String("minio-secret-access-key"),
174
-
Bucket: ctx.String("minio-bucket"),
175
-
Location: ctx.String("minio-location"),
176
-
BasePath: ctx.String("minio-base-path"),
177
-
UseSSL: ctx.Bool("minio-use-ssl"),
180
+
Endpoint: ctx.String("minio-endpoint"),
181
+
AccessKeyID: ctx.String("minio-access-key-id"),
182
+
SecretAccessKey: ctx.String("minio-secret-access-key"),
183
+
Bucket: ctx.String("minio-bucket"),
184
+
Location: ctx.String("minio-location"),
185
+
BasePath: ctx.String("minio-base-path"),
186
+
UseSSL: ctx.Bool("minio-use-ssl"),
187
+
InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"),
188
+
ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"),
178
189
})
179
190
default:
180
191
return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
+6
-3
custom/conf/app.example.ini
+6
-3
custom/conf/app.example.ini
···
583
583
;; * In request Header: X-Request-ID: test-id-123
584
584
;; * Configuration in app.ini: REQUEST_ID_HEADERS = X-Request-ID
585
585
;; * Print in log: 127.0.0.1:58384 - - [14/Feb/2023:16:33:51 +0800] "test-id-123"
586
-
;;
587
-
;; If you configure more than one in the .ini file, it will match in the order of configuration,
586
+
;;
587
+
;; If you configure more than one in the .ini file, it will match in the order of configuration,
588
588
;; and the first match will be finally printed in the log.
589
589
;; * E.g:
590
590
;; * In reuqest Header: X-Trace-ID: trace-id-1q2w3e4r
591
591
;; * Configuration in app.ini: REQUEST_ID_HEADERS = X-Request-ID, X-Trace-ID, X-Req-ID
592
592
;; * Print in log: 127.0.0.1:58384 - - [14/Feb/2023:16:33:51 +0800] "trace-id-1q2w3e4r"
593
593
;;
594
-
;; REQUEST_ID_HEADERS =
594
+
;; REQUEST_ID_HEADERS =
595
595
596
596
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
597
597
;;
···
1886
1886
;;
1887
1887
;; Minio skip SSL verification available when STORAGE_TYPE is `minio`
1888
1888
;MINIO_INSECURE_SKIP_VERIFY = false
1889
+
;;
1890
+
;; Minio checksum algorithm: default (for MinIO or AWS S3) or md5 (for Cloudflare or Backblaze)
1891
+
;MINIO_CHECKSUM_ALGORITHM = default
1889
1892
1890
1893
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1891
1894
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+1
docs/content/doc/administration/config-cheat-sheet.en-us.md
+1
docs/content/doc/administration/config-cheat-sheet.en-us.md
···
855
855
- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`
856
856
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORAGE_TYPE is `minio`
857
857
- `MINIO_INSECURE_SKIP_VERIFY`: **false**: Minio skip SSL verification available when STORAGE_TYPE is `minio`
858
+
- `MINIO_CHECKSUM_ALGORITHM`: **default**: Minio checksum algorithm: `default` (for MinIO or AWS S3) or `md5` (for Cloudflare or Backblaze)
858
859
859
860
## Log (`log`)
860
861
+29
-11
modules/lfs/content_store.go
+29
-11
modules/lfs/content_store.go
···
60
60
return err
61
61
}
62
62
63
-
// This shouldn't happen but it is sensible to test
64
-
if written != pointer.Size {
65
-
if err := s.Delete(p); err != nil {
66
-
log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err)
63
+
// check again whether there is any error during the Save operation
64
+
// because some errors might be ignored by the Reader's caller
65
+
if wrappedRd.lastError != nil && !errors.Is(wrappedRd.lastError, io.EOF) {
66
+
err = wrappedRd.lastError
67
+
} else if written != pointer.Size {
68
+
err = ErrSizeMismatch
69
+
}
70
+
71
+
// if the upload failed, try to delete the file
72
+
if err != nil {
73
+
if errDel := s.Delete(p); errDel != nil {
74
+
log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, errDel)
67
75
}
68
-
return ErrSizeMismatch
69
76
}
70
77
71
-
return nil
78
+
return err
72
79
}
73
80
74
81
// Exists returns true if the object exists in the content store.
···
109
116
expectedSize int64
110
117
hash hash.Hash
111
118
expectedHash string
119
+
lastError error
120
+
}
121
+
122
+
// recordError records the last error during the Save operation
123
+
// Some callers of the Reader doesn't respect the returned "err"
124
+
// For example, MinIO's Put will ignore errors if the written size could equal to expected size
125
+
// So we must remember the error by ourselves,
126
+
// and later check again whether ErrSizeMismatch or ErrHashMismatch occurs during the Save operation
127
+
func (r *hashingReader) recordError(err error) error {
128
+
r.lastError = err
129
+
return err
112
130
}
113
131
114
132
func (r *hashingReader) Read(b []byte) (int, error) {
···
118
136
r.currentSize += int64(n)
119
137
wn, werr := r.hash.Write(b[:n])
120
138
if wn != n || werr != nil {
121
-
return n, werr
139
+
return n, r.recordError(werr)
122
140
}
123
141
}
124
142
125
-
if err != nil && err == io.EOF {
143
+
if errors.Is(err, io.EOF) || r.currentSize >= r.expectedSize {
126
144
if r.currentSize != r.expectedSize {
127
-
return n, ErrSizeMismatch
145
+
return n, r.recordError(ErrSizeMismatch)
128
146
}
129
147
130
148
shaStr := hex.EncodeToString(r.hash.Sum(nil))
131
149
if shaStr != r.expectedHash {
132
-
return n, ErrHashMismatch
150
+
return n, r.recordError(ErrHashMismatch)
133
151
}
134
152
}
135
153
136
-
return n, err
154
+
return n, r.recordError(err)
137
155
}
138
156
139
157
func newHashingReader(expectedSize int64, expectedHash string, reader io.Reader) *hashingReader {
+1
modules/setting/storage.go
+1
modules/setting/storage.go
···
42
42
sec.Key("MINIO_LOCATION").MustString("us-east-1")
43
43
sec.Key("MINIO_USE_SSL").MustBool(false)
44
44
sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
45
+
sec.Key("MINIO_CHECKSUM_ALGORITHM").MustString("default")
45
46
46
47
if targetSec == nil {
47
48
targetSec, _ = rootCfg.NewSection(name)
+18
-3
modules/storage/minio.go
+18
-3
modules/storage/minio.go
···
6
6
import (
7
7
"context"
8
8
"crypto/tls"
9
+
"fmt"
9
10
"io"
10
11
"net/http"
11
12
"net/url"
···
53
54
BasePath string `ini:"MINIO_BASE_PATH"`
54
55
UseSSL bool `ini:"MINIO_USE_SSL"`
55
56
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
57
+
ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM"`
56
58
}
57
59
58
60
// MinioStorage returns a minio bucket storage
59
61
type MinioStorage struct {
62
+
cfg *MinioStorageConfig
60
63
ctx context.Context
61
64
client *minio.Client
62
65
bucket string
···
91
94
}
92
95
config := configInterface.(MinioStorageConfig)
93
96
97
+
if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
98
+
return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
99
+
}
100
+
94
101
log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
95
102
96
103
minioClient, err := minio.New(config.Endpoint, &minio.Options{
···
113
120
}
114
121
115
122
return &MinioStorage{
123
+
cfg: &config,
116
124
ctx: ctx,
117
125
client: minioClient,
118
126
bucket: config.Bucket,
···
124
132
return util.PathJoinRelX(m.basePath, p)
125
133
}
126
134
127
-
// Open open a file
135
+
// Open opens a file
128
136
func (m *MinioStorage) Open(path string) (Object, error) {
129
137
opts := minio.GetObjectOptions{}
130
138
object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
···
134
142
return &minioObject{object}, nil
135
143
}
136
144
137
-
// Save save a file to minio
145
+
// Save saves a file to minio
138
146
func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) {
139
147
uploadInfo, err := m.client.PutObject(
140
148
m.ctx,
···
142
150
m.buildMinioPath(path),
143
151
r,
144
152
size,
145
-
minio.PutObjectOptions{ContentType: "application/octet-stream"},
153
+
minio.PutObjectOptions{
154
+
ContentType: "application/octet-stream",
155
+
// some storages like:
156
+
// * https://developers.cloudflare.com/r2/api/s3/api/
157
+
// * https://www.backblaze.com/b2/docs/s3_compatible_api.html
158
+
// do not support "x-amz-checksum-algorithm" header, so use legacy MD5 checksum
159
+
SendContentMd5: m.cfg.ChecksumAlgorithm == "md5",
160
+
},
146
161
)
147
162
if err != nil {
148
163
return 0, convertMinioErr(err)