forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
lexicons,appview/pulls: upload patches as blobs
By using blobs we massively increase our maximum patch size. When stored
directly on the record patches can be at most 2 MiB. By moving it to a
blob, we get a minimum of 50 MiB (smaller if the pds has set it but 50
is the default).
legacy field name `patch` in `repo.pull` lexicon is preserved (but not
used) to support some level of backward compatibility
Signed-off-by: Samuel Shuert <me@thecoded.prof>
Signed-off-by: Seongmin Lee <git@boltless.me>
authored by
Samuel Shuert
and committed by
tangled.org
4bdc9dfe
e6c79a04
+57
-39
api/tangled/cbor_gen.go
api/tangled/cbor_gen.go
This is a binary file and will not be displayed.
api/tangled/repopull.go
api/tangled/repopull.go
This is a binary file and will not be displayed.
+1
-1
appview/models/pull.go
+1
-1
appview/models/pull.go
···
83
83
Repo *Repo
84
84
}
85
85
86
+
// NOTE: This method does not include patch blob in returned atproto record
86
87
func (p Pull) AsRecord() tangled.RepoPull {
87
88
var source *tangled.RepoPull_Source
88
89
if p.PullSource != nil {
···
114
113
Repo: p.RepoAt.String(),
115
114
Branch: p.TargetBranch,
116
115
},
117
-
Patch: p.LatestPatch(),
118
116
Source: source,
119
117
}
120
118
return record
+48
-36
appview/pulls/pulls.go
+48
-36
appview/pulls/pulls.go
···
1241
1241
return
1242
1242
}
1243
1243
1244
+
blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(patch))
1245
+
if err != nil {
1246
+
log.Println("failed to upload patch", err)
1247
+
s.pages.Notice(w, "pull", "Failed to create pull request. Try again later.")
1248
+
return
1249
+
}
1250
+
1244
1251
_, err = comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{
1245
1252
Collection: tangled.RepoPullNSID,
1246
1253
Repo: user.Did,
···
1259
1252
Repo: string(repo.RepoAt()),
1260
1253
Branch: targetBranch,
1261
1254
},
1262
-
Patch: patch,
1255
+
PatchBlob: blob.Blob,
1263
1256
Source: recordPullSource,
1264
1257
CreatedAt: time.Now().Format(time.RFC3339),
1265
1258
},
···
1335
1328
// apply all record creations at once
1336
1329
var writes []*comatproto.RepoApplyWrites_Input_Writes_Elem
1337
1330
for _, p := range stack {
1331
+
blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(p.LatestPatch()))
1332
+
if err != nil {
1333
+
log.Println("failed to upload patch blob", err)
1334
+
s.pages.Notice(w, "pull", "Failed to create pull request. Try again later.")
1335
+
return
1336
+
}
1337
+
1338
1338
record := p.AsRecord()
1339
-
write := comatproto.RepoApplyWrites_Input_Writes_Elem{
1339
+
record.PatchBlob = blob.Blob
1340
+
writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{
1340
1341
RepoApplyWrites_Create: &comatproto.RepoApplyWrites_Create{
1341
1342
Collection: tangled.RepoPullNSID,
1342
1343
Rkey: &p.Rkey,
···
1352
1337
Val: &record,
1353
1338
},
1354
1339
},
1355
-
}
1356
-
writes = append(writes, &write)
1340
+
})
1357
1341
}
1358
1342
_, err = comatproto.RepoApplyWrites(r.Context(), client, &comatproto.RepoApplyWrites_Input{
1359
1343
Repo: user.Did,
···
1885
1871
return
1886
1872
}
1887
1873
1888
-
var recordPullSource *tangled.RepoPull_Source
1889
-
if pull.IsBranchBased() {
1890
-
recordPullSource = &tangled.RepoPull_Source{
1891
-
Branch: pull.PullSource.Branch,
1892
-
Sha: sourceRev,
1893
-
}
1874
+
blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(patch))
1875
+
if err != nil {
1876
+
log.Println("failed to upload patch blob", err)
1877
+
s.pages.Notice(w, "resubmit-error", "Failed to update pull request on the PDS. Try again later.")
1878
+
return
1894
1879
}
1895
-
if pull.IsForkBased() {
1896
-
repoAt := pull.PullSource.RepoAt.String()
1897
-
recordPullSource = &tangled.RepoPull_Source{
1898
-
Branch: pull.PullSource.Branch,
1899
-
Repo: &repoAt,
1900
-
Sha: sourceRev,
1901
-
}
1902
-
}
1880
+
record := pull.AsRecord()
1881
+
record.PatchBlob = blob.Blob
1882
+
record.CreatedAt = time.Now().Format(time.RFC3339)
1903
1883
1904
1884
_, err = comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{
1905
1885
Collection: tangled.RepoPullNSID,
···
1901
1893
Rkey: pull.Rkey,
1902
1894
SwapRecord: ex.Cid,
1903
1895
Record: &lexutil.LexiconTypeDecoder{
1904
-
Val: &tangled.RepoPull{
1905
-
Title: pull.Title,
1906
-
Target: &tangled.RepoPull_Target{
1907
-
Repo: string(repo.RepoAt()),
1908
-
Branch: pull.TargetBranch,
1909
-
},
1910
-
Patch: patch, // new patch
1911
-
Source: recordPullSource,
1912
-
CreatedAt: time.Now().Format(time.RFC3339),
1913
-
},
1896
+
Val: &record,
1914
1897
},
1915
1898
})
1916
1899
if err != nil {
···
1987
1988
}
1988
1989
defer tx.Rollback()
1989
1990
1991
+
client, err := s.oauth.AuthorizedClient(r)
1992
+
if err != nil {
1993
+
log.Println("failed to authorize client")
1994
+
s.pages.Notice(w, "resubmit-error", "Failed to create pull request. Try again later.")
1995
+
return
1996
+
}
1997
+
1990
1998
// pds updates to make
1991
1999
var writes []*comatproto.RepoApplyWrites_Input_Writes_Elem
1992
2000
···
2027
2021
return
2028
2022
}
2029
2023
2024
+
blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(patch))
2025
+
if err != nil {
2026
+
log.Println("failed to upload patch blob", err)
2027
+
s.pages.Notice(w, "resubmit-error", "Failed to update pull request on the PDS. Try again later.")
2028
+
return
2029
+
}
2030
2030
record := p.AsRecord()
2031
+
record.PatchBlob = blob.Blob
2031
2032
writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{
2032
2033
RepoApplyWrites_Create: &comatproto.RepoApplyWrites_Create{
2033
2034
Collection: tangled.RepoPullNSID,
···
2069
2056
return
2070
2057
}
2071
2058
2059
+
blob, err := comatproto.RepoUploadBlob(r.Context(), client, strings.NewReader(patch))
2060
+
if err != nil {
2061
+
log.Println("failed to upload patch blob", err)
2062
+
s.pages.Notice(w, "resubmit-error", "Failed to update pull request on the PDS. Try again later.")
2063
+
return
2064
+
}
2072
2065
record := np.AsRecord()
2073
-
2066
+
record.PatchBlob = blob.Blob
2074
2067
writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{
2075
2068
RepoApplyWrites_Update: &comatproto.RepoApplyWrites_Update{
2076
2069
Collection: tangled.RepoPullNSID,
···
2110
2091
if err != nil {
2111
2092
log.Println("failed to resubmit pull", err)
2112
2093
s.pages.Notice(w, "pull-resubmit-error", "Failed to resubmit pull request. Try again later.")
2113
-
return
2114
-
}
2115
-
2116
-
client, err := s.oauth.AuthorizedClient(r)
2117
-
if err != nil {
2118
-
log.Println("failed to authorize client")
2119
-
s.pages.Notice(w, "resubmit-error", "Failed to create pull request. Try again later.")
2120
2094
return
2121
2095
}
2122
2096
+8
-2
lexicons/pulls/pull.json
+8
-2
lexicons/pulls/pull.json
···
12
12
"required": [
13
13
"target",
14
14
"title",
15
-
"patch",
15
+
"patchBlob",
16
16
"createdAt"
17
17
],
18
18
"properties": {
···
27
27
"type": "string"
28
28
},
29
29
"patch": {
30
-
"type": "string"
30
+
"type": "string",
31
+
"description": "(deprecated) use patchBlob instead"
32
+
},
33
+
"patchBlob": {
34
+
"type": "blob",
35
+
"accept": "text/x-patch",
36
+
"description": "patch content"
31
37
},
32
38
"source": {
33
39
"type": "ref",