@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add support for partially uploaded files
Summary:
Ref T7149. This flags allocated but incomplete files and doesn't explode when trying to download them.
Files are marked complete when the last chunk is uploaded.
I added a key on `<authorPHID, isPartial>` so we can show you a list of partially uploaded files and prompt you to resume them at some point down the road.
Test Plan: Massaged debugging settings and uploaded README.md very slowly in 32b chunks. Saw the file lose its "Partial" flag when the last chunk finished.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: joshuaspence, epriestley
Maniphest Tasks: T7149
Differential Revision: https://secure.phabricator.com/D12063
+136
-58
+2
resources/sql/autopatches/20150312.filechunk.2.sql
+2
resources/sql/autopatches/20150312.filechunk.2.sql
+2
resources/sql/autopatches/20150312.filechunk.3.sql
+2
resources/sql/autopatches/20150312.filechunk.3.sql
+3
-3
src/applications/files/conduit/FileAllocateConduitAPIMethod.php
+3
-3
src/applications/files/conduit/FileAllocateConduitAPIMethod.php
···
53
53
$hash,
54
54
$properties);
55
55
56
-
if ($file && !$force_chunking) {
56
+
if ($file) {
57
57
return array(
58
58
'upload' => false,
59
59
'filePHID' => $file->getPHID(),
···
70
70
71
71
if ($file) {
72
72
return array(
73
-
'upload' => $file->isPartial(),
73
+
'upload' => (bool)$file->getIsPartial(),
74
74
'filePHID' => $file->getPHID(),
75
75
);
76
76
}
···
103
103
// Otherwise, this is a large file and we need to perform a chunked
104
104
// upload.
105
105
106
-
$chunk_properties = array();
106
+
$chunk_properties = $properties;
107
107
108
108
if ($hash) {
109
109
$chunk_properties += array(
+11
src/applications/files/conduit/FileConduitAPIMethod.php
+11
src/applications/files/conduit/FileConduitAPIMethod.php
···
89
89
return $data;
90
90
}
91
91
92
+
protected function loadAnyMissingChunk(
93
+
PhabricatorUser $viewer,
94
+
PhabricatorFile $file) {
95
+
96
+
return $this->newChunkQuery($viewer, $file)
97
+
->withIsComplete(false)
98
+
->setLimit(1)
99
+
->execute();
100
+
}
101
+
92
102
private function newChunkQuery(
93
103
PhabricatorUser $viewer,
94
104
PhabricatorFile $file) {
···
105
115
->setViewer($viewer)
106
116
->withChunkHandles(array($file->getStorageHandle()));
107
117
}
118
+
108
119
109
120
}
+6
-3
src/applications/files/conduit/FileUploadChunkConduitAPIMethod.php
+6
-3
src/applications/files/conduit/FileUploadChunkConduitAPIMethod.php
···
57
57
58
58
// NOTE: These files have a view policy which prevents normal access. They
59
59
// are only accessed through the storage engine.
60
-
$file = PhabricatorFile::newFromFileData(
60
+
$chunk_data = PhabricatorFile::newFromFileData(
61
61
$data,
62
62
array(
63
63
'name' => $file->getMonogram().'.chunk-'.$chunk->getID(),
64
64
'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
65
65
));
66
66
67
-
$chunk->setDataFilePHID($file->getPHID())->save();
67
+
$chunk->setDataFilePHID($chunk_data->getPHID())->save();
68
68
69
-
// TODO: If all chunks are up, mark the file as complete.
69
+
$missing = $this->loadAnyMissingChunk($viewer, $file);
70
+
if (!$missing) {
71
+
$file->setIsPartial(0)->save();
72
+
}
70
73
71
74
return null;
72
75
}
+57
-40
src/applications/files/controller/PhabricatorFileDataController.php
+57
-40
src/applications/files/controller/PhabricatorFileDataController.php
···
5
5
private $phid;
6
6
private $key;
7
7
private $token;
8
+
private $file;
8
9
9
10
public function willProcessRequest(array $data) {
10
11
$this->phid = $data['phid'];
···
16
17
return false;
17
18
}
18
19
19
-
protected function checkFileAndToken($file) {
20
-
if (!$file) {
21
-
return new Aphront404Response();
22
-
}
23
-
24
-
if (!$file->validateSecretKey($this->key)) {
25
-
return new Aphront403Response();
26
-
}
27
-
28
-
return null;
29
-
}
30
-
31
20
public function processRequest() {
32
21
$request = $this->getRequest();
22
+
$viewer = $this->getViewer();
33
23
34
24
$alt = PhabricatorEnv::getEnvConfig('security.alternate-file-domain');
35
25
$base_uri = PhabricatorEnv::getEnvConfig('phabricator.base-uri');
···
44
34
// Alternate files domain isn't configured or it's set
45
35
// to the same as the default domain
46
36
47
-
// load the file with permissions checks;
48
-
$file = id(new PhabricatorFileQuery())
49
-
->setViewer($request->getUser())
50
-
->withPHIDs(array($this->phid))
51
-
->executeOne();
52
-
53
-
$error_response = $this->checkFileAndToken($file);
54
-
if ($error_response) {
55
-
return $error_response;
37
+
$response = $this->loadFile($viewer);
38
+
if ($response) {
39
+
return $response;
56
40
}
41
+
$file = $this->getFile();
57
42
58
43
// when the file is not CDNable, don't allow cache
59
44
$cache_response = $file->getCanCDN();
60
45
} else if ($req_domain != $alt_domain) {
61
46
// Alternate domain is configured but this request isn't using it
62
47
63
-
// load the file with permissions checks;
64
-
$file = id(new PhabricatorFileQuery())
65
-
->setViewer($request->getUser())
66
-
->withPHIDs(array($this->phid))
67
-
->executeOne();
68
-
69
-
$error_response = $this->checkFileAndToken($file);
70
-
if ($error_response) {
71
-
return $error_response;
48
+
$response = $this->loadFile($viewer);
49
+
if ($response) {
50
+
return $response;
72
51
}
52
+
$file = $this->getFile();
73
53
74
54
// if the user can see the file, generate a token;
75
55
// redirect to the alt domain with the token;
···
82
62
->setURI($token_uri);
83
63
84
64
} else {
85
-
// We are using the alternate domain
86
-
87
-
// load the file, bypassing permission checks;
88
-
$file = id(new PhabricatorFileQuery())
89
-
->setViewer(PhabricatorUser::getOmnipotentUser())
90
-
->withPHIDs(array($this->phid))
91
-
->executeOne();
65
+
// We are using the alternate domain. We don't have authentication
66
+
// on this domain, so we bypass policy checks when loading the file.
92
67
93
-
$error_response = $this->checkFileAndToken($file);
94
-
if ($error_response) {
95
-
return $error_response;
68
+
$bypass_policies = PhabricatorUser::getOmnipotentUser();
69
+
$response = $this->loadFile($bypass_policies);
70
+
if ($response) {
71
+
return $response;
96
72
}
73
+
$file = $this->getFile();
97
74
98
75
$acquire_token_uri = id(new PhutilURI($file->getViewURI()))
99
76
->setDomain($main_domain);
···
203
180
}
204
181
205
182
return $uri;
183
+
}
184
+
185
+
private function loadFile(PhabricatorUser $viewer) {
186
+
$file = id(new PhabricatorFileQuery())
187
+
->setViewer($viewer)
188
+
->withPHIDs(array($this->phid))
189
+
->executeOne();
190
+
191
+
if (!$file) {
192
+
return new Aphront404Response();
193
+
}
194
+
195
+
if (!$file->validateSecretKey($this->key)) {
196
+
return new Aphront403Response();
197
+
}
198
+
199
+
if ($file->getIsPartial()) {
200
+
// We may be on the CDN domain, so we need to use a fully-qualified URI
201
+
// here to make sure we end up back on the main domain.
202
+
$info_uri = PhabricatorEnv::getURI($file->getInfoURI());
203
+
204
+
return $this->newDialog()
205
+
->setTitle(pht('Partial Upload'))
206
+
->appendParagraph(
207
+
pht(
208
+
'This file has only been partially uploaded. It must be '.
209
+
'uploaded completely before you can download it.'))
210
+
->addCancelButton($info_uri);
211
+
}
212
+
213
+
$this->file = $file;
214
+
215
+
return null;
216
+
}
217
+
218
+
private function getFile() {
219
+
if (!$this->file) {
220
+
throw new Exception(pht('Call loadFile() before getFile()!'));
221
+
}
222
+
return $this->file;
206
223
}
207
224
208
225
}
+21
-5
src/applications/files/controller/PhabricatorFileInfoController.php
+21
-5
src/applications/files/controller/PhabricatorFileInfoController.php
···
52
52
$ttl = $file->getTTL();
53
53
if ($ttl !== null) {
54
54
$ttl_tag = id(new PHUITagView())
55
-
->setType(PHUITagView::TYPE_OBJECT)
55
+
->setType(PHUITagView::TYPE_STATE)
56
+
->setBackgroundColor(PHUITagView::COLOR_YELLOW)
56
57
->setName(pht('Temporary'));
57
58
$header->addTag($ttl_tag);
59
+
}
60
+
61
+
$partial = $file->getIsPartial();
62
+
if ($partial) {
63
+
$partial_tag = id(new PHUITagView())
64
+
->setType(PHUITagView::TYPE_STATE)
65
+
->setBackgroundColor(PHUITagView::COLOR_ORANGE)
66
+
->setName(pht('Partial Upload'));
67
+
$header->addTag($partial_tag);
58
68
}
59
69
60
70
$actions = $this->buildActionView($file);
···
126
136
->setObjectURI($this->getRequest()->getRequestURI())
127
137
->setObject($file);
128
138
139
+
$can_download = !$file->getIsPartial();
140
+
129
141
if ($file->isViewableInBrowser()) {
130
142
$view->addAction(
131
143
id(new PhabricatorActionView())
132
144
->setName(pht('View File'))
133
145
->setIcon('fa-file-o')
134
-
->setHref($file->getViewURI()));
146
+
->setHref($file->getViewURI())
147
+
->setDisabled(!$can_download)
148
+
->setWorkflow(!$can_download));
135
149
} else {
136
150
$view->addAction(
137
151
id(new PhabricatorActionView())
138
152
->setUser($viewer)
139
-
->setRenderAsForm(true)
140
-
->setDownload(true)
153
+
->setRenderAsForm($can_download)
154
+
->setDownload($can_download)
141
155
->setName(pht('Download File'))
142
156
->setIcon('fa-download')
143
-
->setHref($file->getViewURI()));
157
+
->setHref($file->getViewURI())
158
+
->setDisabled(!$can_download)
159
+
->setWorkflow(!$can_download));
144
160
}
145
161
146
162
$view->addAction(
+18
src/applications/files/query/PhabricatorFileChunkQuery.php
+18
src/applications/files/query/PhabricatorFileChunkQuery.php
···
6
6
private $chunkHandles;
7
7
private $rangeStart;
8
8
private $rangeEnd;
9
+
private $isComplete;
9
10
private $needDataFiles;
10
11
11
12
public function withChunkHandles(array $handles) {
···
16
17
public function withByteRange($start, $end) {
17
18
$this->rangeStart = $start;
18
19
$this->rangeEnd = $end;
20
+
return $this;
21
+
}
22
+
23
+
public function withIsComplete($complete) {
24
+
$this->isComplete = $complete;
19
25
return $this;
20
26
}
21
27
···
102
108
$conn_r,
103
109
'byteStart < %d',
104
110
$this->rangeEnd);
111
+
}
112
+
113
+
if ($this->isComplete !== null) {
114
+
if ($this->isComplete) {
115
+
$where[] = qsprintf(
116
+
$conn_r,
117
+
'dataFilePHID IS NOT NULL');
118
+
} else {
119
+
$where[] = qsprintf(
120
+
$conn_r,
121
+
'dataFilePHID IS NULL');
122
+
}
105
123
}
106
124
107
125
$where[] = $this->buildPagingClause($conn_r);
+16
-7
src/applications/files/storage/PhabricatorFile.php
+16
-7
src/applications/files/storage/PhabricatorFile.php
···
51
51
protected $ttl;
52
52
protected $isExplicitUpload = 1;
53
53
protected $viewPolicy = PhabricatorPolicies::POLICY_USER;
54
+
protected $isPartial = 0;
54
55
55
56
private $objects = self::ATTACHABLE;
56
57
private $objectPHIDs = self::ATTACHABLE;
···
67
68
68
69
return id(new PhabricatorFile())
69
70
->setViewPolicy($view_policy)
71
+
->setIsPartial(0)
70
72
->attachOriginalFile(null)
71
73
->attachObjects(array())
72
74
->attachObjectPHIDs(array());
···
91
93
'ttl' => 'epoch?',
92
94
'isExplicitUpload' => 'bool?',
93
95
'mailKey' => 'bytes20',
96
+
'isPartial' => 'bool',
94
97
),
95
98
self::CONFIG_KEY_SCHEMA => array(
96
99
'key_phid' => null,
···
109
112
),
110
113
'key_dateCreated' => array(
111
114
'columns' => array('dateCreated'),
115
+
),
116
+
'key_partial' => array(
117
+
'columns' => array('authorPHID', 'isPartial'),
112
118
),
113
119
),
114
120
) + parent::getConfiguration();
···
294
300
$file->setStorageEngine($engine->getEngineIdentifier());
295
301
$file->setStorageHandle(PhabricatorFileChunk::newChunkHandle());
296
302
$file->setStorageFormat(self::STORAGE_FORMAT_RAW);
303
+
$file->setIsPartial(1);
297
304
298
305
$file->readPropertiesFromParameters($params);
299
306
···
628
635
}
629
636
$parts[] = $name;
630
637
631
-
$path = implode('/', $parts);
638
+
$path = '/'.implode('/', $parts);
632
639
633
-
return PhabricatorEnv::getCDNURI($path);
640
+
// If this file is only partially uploaded, we're just going to return a
641
+
// local URI to make sure that Ajax works, since the page is inevitably
642
+
// going to give us an error back.
643
+
if ($this->getIsPartial()) {
644
+
return PhabricatorEnv::getURI($path);
645
+
} else {
646
+
return PhabricatorEnv::getCDNURI($path);
647
+
}
634
648
}
635
649
636
650
/**
···
1168
1182
return id(new AphrontRedirectResponse())
1169
1183
->setIsExternal($is_external)
1170
1184
->setURI($uri);
1171
-
}
1172
-
1173
-
public function isPartial() {
1174
-
// TODO: Placeholder for resumable uploads.
1175
-
return false;
1176
1185
}
1177
1186
1178
1187