@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.) hq.recaptime.dev/wiki/Phorge
phorge phabricator
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Make `diffusion.filecontentquery` return file PHIDs instead of raw content

Summary:
Fixes T9319. Proxied requests (e.g., in the cluster) for binary files (like images) currently fail because we can not return binary data over Conduit in JSON.

Although Conduit will eventually support binary-safe encodings, a cleaner approach to this is just to return a `filePHID` instead of the raw content. This is generally faster and more flexible, and gives us more opportunities to add caching later.

After making the call, the client pulls the file data separately.

We also no longer need to return a complex data structure because we don't do blame over this call any longer.

Test Plan:
- Viewed images in Diffusion.
- Viewed READMEs in Diffusion.
- Used `bin/differential attach-commit rX Dy` to hit attach pathway.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T9319

Differential Revision: https://secure.phabricator.com/D14970

+183 -253
-2
src/__phutil_library_map__.php
··· 598 598 'DiffusionExternalController' => 'applications/diffusion/controller/DiffusionExternalController.php', 599 599 'DiffusionExternalSymbolQuery' => 'applications/diffusion/symbol/DiffusionExternalSymbolQuery.php', 600 600 'DiffusionExternalSymbolsSource' => 'applications/diffusion/symbol/DiffusionExternalSymbolsSource.php', 601 - 'DiffusionFileContent' => 'applications/diffusion/data/DiffusionFileContent.php', 602 601 'DiffusionFileContentQuery' => 'applications/diffusion/query/filecontent/DiffusionFileContentQuery.php', 603 602 'DiffusionFileContentQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php', 604 603 'DiffusionFindSymbolsConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionFindSymbolsConduitAPIMethod.php', ··· 4571 4570 'DiffusionExternalController' => 'DiffusionController', 4572 4571 'DiffusionExternalSymbolQuery' => 'Phobject', 4573 4572 'DiffusionExternalSymbolsSource' => 'Phobject', 4574 - 'DiffusionFileContent' => 'Phobject', 4575 4573 'DiffusionFileContentQuery' => 'DiffusionQuery', 4576 4574 'DiffusionFileContentQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod', 4577 4575 'DiffusionFindSymbolsConduitAPIMethod' => 'DiffusionConduitAPIMethod',
+14 -2
src/applications/differential/engine/DifferentialDiffExtractionEngine.php
··· 156 156 'commit' => $identifier, 157 157 'path' => $path, 158 158 )); 159 - $corpus = $response['corpus']; 160 159 161 - if ($files[$file_phid]->loadFileData() != $corpus) { 160 + $new_file_phid = $response['filePHID']; 161 + if (!$new_file_phid) { 162 + return true; 163 + } 164 + 165 + $new_file = id(new PhabricatorFileQuery()) 166 + ->setViewer($viewer) 167 + ->withPHIDs(array($new_file_phid)) 168 + ->executeOne(); 169 + if (!$new_file) { 170 + return true; 171 + } 172 + 173 + if ($files[$file_phid]->loadFileData() != $new_file->loadFileData()) { 162 174 return true; 163 175 } 164 176 } else {
+36 -9
src/applications/diffusion/conduit/DiffusionFileContentQueryConduitAPIMethod.php
··· 27 27 protected function getResult(ConduitAPIRequest $request) { 28 28 $drequest = $this->getDiffusionRequest(); 29 29 30 - $file_query = DiffusionFileContentQuery::newFromDiffusionRequest($drequest) 31 - ->setViewer($request->getUser()); 30 + $file_query = DiffusionFileContentQuery::newFromDiffusionRequest($drequest); 32 31 33 32 $timeout = $request->getValue('timeout'); 34 33 if ($timeout) { ··· 40 39 $file_query->setByteLimit($byte_limit); 41 40 } 42 41 43 - $file_content = $file_query->loadFileContent(); 42 + $content = $file_query->execute(); 43 + 44 + $too_slow = (bool)$file_query->getExceededTimeLimit(); 45 + $too_huge = (bool)$file_query->getExceededByteLimit(); 46 + 47 + $file_phid = null; 48 + if (!$too_slow && !$too_huge) { 49 + $file = $this->newFile($drequest, $content); 50 + $file_phid = $file->getPHID(); 51 + } 52 + 53 + return array( 54 + 'tooSlow' => $too_slow, 55 + 'tooHuge' => $too_huge, 56 + 'filePHID' => $file_phid, 57 + ); 58 + } 59 + 60 + private function newFile(DiffusionRequest $drequest, $content) { 61 + $path = $drequest->getPath(); 62 + $name = basename($path); 44 63 45 - $text_list = $rev_list = $blame_dict = array(); 64 + $repository = $drequest->getRepository(); 65 + $repository_phid = $repository->getPHID(); 46 66 47 - $file_content 48 - ->setBlameDict($blame_dict) 49 - ->setRevList($rev_list) 50 - ->setTextList($text_list); 67 + $file = PhabricatorFile::buildFromFileDataOrHash( 68 + $content, 69 + array( 70 + 'name' => $name, 71 + 'ttl' => time() + phutil_units('48 hours in seconds'), 72 + 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, 73 + )); 51 74 52 - return $file_content->toDictionary(); 75 + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); 76 + $file->attachToObject($repository_phid); 77 + unset($unguarded); 78 + 79 + return $file; 53 80 } 54 81 55 82 }
+49 -71
src/applications/diffusion/controller/DiffusionBrowseController.php
··· 153 153 ); 154 154 } 155 155 156 - $file_content = DiffusionFileContent::newFromConduit( 157 - $this->callConduitWithDiffusionRequest( 158 - 'diffusion.filecontentquery', 159 - $params)); 160 - $data = $file_content->getCorpus(); 156 + $response = $this->callConduitWithDiffusionRequest( 157 + 'diffusion.filecontentquery', 158 + $params); 161 159 162 - if ($view === 'raw') { 163 - return $this->buildRawResponse($path, $data); 164 - } 160 + $hit_byte_limit = $response['tooHuge']; 161 + $hit_time_limit = $response['tooSlow']; 165 162 166 - $this->loadLintMessages(); 167 - $this->coverage = $drequest->loadCoverage(); 168 - 169 - if ($byte_limit && (strlen($data) == $byte_limit)) { 163 + $file_phid = $response['filePHID']; 164 + if ($hit_byte_limit) { 170 165 $corpus = $this->buildErrorCorpus( 171 166 pht( 172 167 'This file is larger than %s byte(s), and too large to display '. 173 168 'in the web UI.', 174 - $byte_limit)); 175 - } else if (ArcanistDiffUtils::isHeuristicBinaryFile($data)) { 176 - $file = $this->loadFileForData($path, $data); 177 - $file_uri = $file->getBestURI(); 169 + phutil_format_bytes($byte_limit))); 170 + } else if ($hit_time_limit) { 171 + $corpus = $this->buildErrorCorpus( 172 + pht( 173 + 'This file took too long to load from the repository (more than '. 174 + '%s second(s)).', 175 + new PhutilNumber($time_limit))); 176 + } else { 177 + $file = id(new PhabricatorFileQuery()) 178 + ->setViewer($viewer) 179 + ->withPHIDs(array($file_phid)) 180 + ->executeOne(); 181 + if (!$file) { 182 + throw new Exception(pht('Failed to load content file!')); 183 + } 178 184 179 - if ($file->isViewableImage()) { 180 - $corpus = $this->buildImageCorpus($file_uri); 185 + if ($view === 'raw') { 186 + return $file->getRedirectResponse(); 187 + } 188 + 189 + $data = $file->loadFileData(); 190 + if (ArcanistDiffUtils::isHeuristicBinaryFile($data)) { 191 + $file_uri = $file->getBestURI(); 192 + 193 + if ($file->isViewableImage()) { 194 + $corpus = $this->buildImageCorpus($file_uri); 195 + } else { 196 + $corpus = $this->buildBinaryCorpus($file_uri, $data); 197 + } 181 198 } else { 182 - $corpus = $this->buildBinaryCorpus($file_uri, $data); 199 + $this->loadLintMessages(); 200 + $this->coverage = $drequest->loadCoverage(); 201 + 202 + // Build the content of the file. 203 + $corpus = $this->buildCorpus( 204 + $show_blame, 205 + $show_color, 206 + $data, 207 + $needs_blame, 208 + $drequest, 209 + $path, 210 + $data); 183 211 } 184 - } else { 185 - // Build the content of the file. 186 - $corpus = $this->buildCorpus( 187 - $show_blame, 188 - $show_color, 189 - $file_content, 190 - $needs_blame, 191 - $drequest, 192 - $path, 193 - $data); 194 212 } 195 213 196 214 if ($request->isAjax()) { ··· 327 345 } 328 346 329 347 $content[] = $this->buildOpenRevisions(); 330 - 331 - 332 - $readme_path = $results->getReadmePath(); 333 - if ($readme_path) { 334 - $readme_content = $this->callConduitWithDiffusionRequest( 335 - 'diffusion.filecontentquery', 336 - array( 337 - 'path' => $readme_path, 338 - 'commit' => $drequest->getStableCommit(), 339 - )); 340 - if ($readme_content) { 341 - $content[] = id(new DiffusionReadmeView()) 342 - ->setUser($this->getViewer()) 343 - ->setPath($readme_path) 344 - ->setContent($readme_content['corpus']); 345 - } 346 - } 348 + $content[] = $this->renderDirectoryReadme($results); 347 349 348 350 $crumbs = $this->buildCrumbs( 349 351 array( ··· 584 586 private function buildCorpus( 585 587 $show_blame, 586 588 $show_color, 587 - DiffusionFileContent $file_content, 589 + $file_corpus, 588 590 $needs_blame, 589 591 DiffusionRequest $drequest, 590 592 $path, ··· 594 596 $blame_timeout = 15; 595 597 $blame_failed = false; 596 598 597 - $file_corpus = $file_content->getCorpus(); 598 599 $highlight_limit = DifferentialChangesetParser::HIGHLIGHT_BYTE_LIMIT; 599 600 $blame_limit = DifferentialChangesetParser::HIGHLIGHT_BYTE_LIMIT; 600 601 $can_highlight = (strlen($file_corpus) <= $highlight_limit); ··· 1256 1257 return $rows; 1257 1258 } 1258 1259 1259 - private function loadFileForData($path, $data) { 1260 - $file = PhabricatorFile::buildFromFileDataOrHash( 1261 - $data, 1262 - array( 1263 - 'name' => basename($path), 1264 - 'ttl' => time() + 60 * 60 * 24, 1265 - 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, 1266 - )); 1267 - 1268 - $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); 1269 - $file->attachToObject( 1270 - $this->getDiffusionRequest()->getRepository()->getPHID()); 1271 - unset($unguarded); 1272 - 1273 - return $file; 1274 - } 1275 - 1276 - private function buildRawResponse($path, $data) { 1277 - $file = $this->loadFileForData($path, $data); 1278 - return $file->getRedirectResponse(); 1279 - } 1280 - 1281 1260 private function buildImageCorpus($file_uri) { 1282 1261 $properties = new PHUIPropertyListView(); 1283 1262 ··· 1299 1278 } 1300 1279 1301 1280 private function buildBinaryCorpus($file_uri, $data) { 1302 - 1303 1281 $size = new PhutilNumber(strlen($data)); 1304 1282 $text = pht('This is a binary file. It is %s byte(s) in length.', $size); 1305 1283 $text = id(new PHUIBoxView())
+45
src/applications/diffusion/controller/DiffusionController.php
··· 287 287 ->appendChild($pager); 288 288 } 289 289 290 + protected function renderDirectoryReadme(DiffusionBrowseResultSet $browse) { 291 + $readme_path = $browse->getReadmePath(); 292 + if ($readme_path === null) { 293 + return null; 294 + } 295 + 296 + $drequest = $this->getDiffusionRequest(); 297 + $viewer = $this->getViewer(); 298 + 299 + try { 300 + $result = $this->callConduitWithDiffusionRequest( 301 + 'diffusion.filecontentquery', 302 + array( 303 + 'path' => $readme_path, 304 + 'commit' => $drequest->getStableCommit(), 305 + )); 306 + } catch (Exception $ex) { 307 + return null; 308 + } 309 + 310 + $file_phid = $result['filePHID']; 311 + if (!$file_phid) { 312 + return null; 313 + } 314 + 315 + $file = id(new PhabricatorFileQuery()) 316 + ->setViewer($viewer) 317 + ->withPHIDs(array($file_phid)) 318 + ->executeOne(); 319 + if (!$file) { 320 + return null; 321 + } 322 + 323 + $corpus = $file->loadFileData(); 324 + 325 + if (!strlen($corpus)) { 326 + return null; 327 + } 328 + 329 + return id(new DiffusionReadmeView()) 330 + ->setUser($this->getViewer()) 331 + ->setPath($readme_path) 332 + ->setContent($corpus); 333 + } 334 + 290 335 }
+3 -16
src/applications/diffusion/controller/DiffusionRepositoryController.php
··· 161 161 $phids = array_keys($phids); 162 162 $handles = $this->loadViewerHandles($phids); 163 163 164 - $readme = null; 165 164 if ($browse_results) { 166 - $readme_path = $browse_results->getReadmePath(); 167 - if ($readme_path) { 168 - $readme_content = $this->callConduitWithDiffusionRequest( 169 - 'diffusion.filecontentquery', 170 - array( 171 - 'path' => $readme_path, 172 - 'commit' => $drequest->getStableCommit(), 173 - )); 174 - if ($readme_content) { 175 - $readme = id(new DiffusionReadmeView()) 176 - ->setUser($this->getViewer()) 177 - ->setPath($readme_path) 178 - ->setContent($readme_content['corpus']); 179 - } 180 - } 165 + $readme = $this->renderDirectoryReadme($browse_results); 166 + } else { 167 + $readme = null; 181 168 } 182 169 183 170 $content[] = $this->buildBrowseTable(
-63
src/applications/diffusion/data/DiffusionFileContent.php
··· 1 - <?php 2 - 3 - final class DiffusionFileContent extends Phobject { 4 - 5 - private $corpus; 6 - private $blameDict; 7 - private $revList; 8 - private $textList; 9 - 10 - public function setTextList(array $text_list) { 11 - $this->textList = $text_list; 12 - return $this; 13 - } 14 - public function getTextList() { 15 - if (!$this->textList) { 16 - return phutil_split_lines($this->getCorpus(), $retain_ends = false); 17 - } 18 - return $this->textList; 19 - } 20 - 21 - public function setRevList(array $rev_list) { 22 - $this->revList = $rev_list; 23 - return $this; 24 - } 25 - public function getRevList() { 26 - return $this->revList; 27 - } 28 - 29 - public function setBlameDict(array $blame_dict) { 30 - $this->blameDict = $blame_dict; 31 - return $this; 32 - } 33 - public function getBlameDict() { 34 - return $this->blameDict; 35 - } 36 - 37 - public function setCorpus($corpus) { 38 - $this->corpus = $corpus; 39 - return $this; 40 - } 41 - 42 - public function getCorpus() { 43 - return $this->corpus; 44 - } 45 - 46 - public function toDictionary() { 47 - return array( 48 - 'corpus' => $this->getCorpus(), 49 - 'blameDict' => $this->getBlameDict(), 50 - 'revList' => $this->getRevList(), 51 - 'textList' => $this->getTextList(), 52 - ); 53 - } 54 - 55 - public static function newFromConduit(array $dict) { 56 - return id(new DiffusionFileContent()) 57 - ->setCorpus($dict['corpus']) 58 - ->setBlameDict($dict['blameDict']) 59 - ->setRevList($dict['revList']) 60 - ->setTextList($dict['textList']); 61 - } 62 - 63 - }
+26 -51
src/applications/diffusion/query/filecontent/DiffusionFileContentQuery.php
··· 1 1 <?php 2 2 3 - /** 4 - * NOTE: this class should only be used where local access to the repository 5 - * is guaranteed and NOT from within the Diffusion application. Diffusion 6 - * should use Conduit method 'diffusion.filecontentquery' to get this sort 7 - * of data. 8 - */ 9 3 abstract class DiffusionFileContentQuery extends DiffusionQuery { 10 4 11 - private $fileContent; 12 - private $viewer; 13 5 private $timeout; 14 6 private $byteLimit; 7 + 8 + private $didHitByteLimit = false; 9 + private $didHitTimeLimit = false; 15 10 16 11 public function setTimeout($timeout) { 17 12 $this->timeout = $timeout; ··· 31 26 return $this->byteLimit; 32 27 } 33 28 34 - public function setViewer(PhabricatorUser $user) { 35 - $this->viewer = $user; 36 - return $this; 29 + final public static function newFromDiffusionRequest( 30 + DiffusionRequest $request) { 31 + return parent::newQueryObject(__CLASS__, $request); 37 32 } 38 33 39 - public function getViewer() { 40 - return $this->viewer; 34 + final public function getExceededByteLimit() { 35 + return $this->didHitByteLimit; 41 36 } 42 37 43 - final public static function newFromDiffusionRequest( 44 - DiffusionRequest $request) { 45 - return parent::newQueryObject(__CLASS__, $request); 38 + final public function getExceededTimeLimit() { 39 + return $this->didHitTimeLimit; 46 40 } 47 41 48 - abstract public function getFileContentFuture(); 49 - abstract protected function executeQueryFromFuture(Future $future); 42 + abstract protected function getFileContentFuture(); 43 + abstract protected function resolveFileContentFuture(Future $future); 50 44 51 - final public function loadFileContentFromFuture(Future $future) { 45 + final protected function executeQuery() { 46 + $future = $this->getFileContentFuture(); 52 47 53 - if ($this->timeout) { 54 - $future->setTimeout($this->timeout); 48 + if ($this->getTimeout()) { 49 + $future->setTimeout($this->getTimeout()); 55 50 } 56 51 57 - if ($this->getByteLimit()) { 58 - $future->setStdoutSizeLimit($this->getByteLimit()); 52 + $byte_limit = $this->getByteLimit(); 53 + if ($byte_limit) { 54 + $future->setStdoutSizeLimit($byte_limit + 1); 59 55 } 60 56 61 57 try { 62 - $file_content = $this->executeQueryFromFuture($future); 58 + $file_content = $this->resolveFileContentFuture($future); 63 59 } catch (CommandException $ex) { 64 60 if (!$future->getWasKilledByTimeout()) { 65 61 throw $ex; 66 62 } 67 63 68 - $message = pht( 69 - '<Attempt to load this file was terminated after %s second(s).>', 70 - $this->timeout); 71 - 72 - $file_content = new DiffusionFileContent(); 73 - $file_content->setCorpus($message); 64 + $this->didHitTimeLimit = true; 65 + $file_content = null; 74 66 } 75 67 76 - $this->fileContent = $file_content; 77 - 78 - $repository = $this->getRequest()->getRepository(); 79 - $try_encoding = $repository->getDetail('encoding'); 80 - if ($try_encoding) { 81 - $this->fileContent->setCorpus( 82 - phutil_utf8_convert( 83 - $this->fileContent->getCorpus(), 'UTF-8', $try_encoding)); 68 + if ($byte_limit && (strlen($file_content) > $byte_limit)) { 69 + $this->didHitByteLimit = true; 70 + $file_content = null; 84 71 } 85 72 86 - return $this->fileContent; 87 - } 88 - 89 - final protected function executeQuery() { 90 - return $this->loadFileContentFromFuture($this->getFileContentFuture()); 91 - } 92 - 93 - final public function loadFileContent() { 94 - return $this->executeQuery(); 95 - } 96 - 97 - final public function getRawData() { 98 - return $this->fileContent->getCorpus(); 73 + return $file_content; 99 74 } 100 75 101 76 }
+3 -7
src/applications/diffusion/query/filecontent/DiffusionGitFileContentQuery.php
··· 2 2 3 3 final class DiffusionGitFileContentQuery extends DiffusionFileContentQuery { 4 4 5 - public function getFileContentFuture() { 5 + protected function getFileContentFuture() { 6 6 $drequest = $this->getRequest(); 7 7 8 8 $repository = $drequest->getRepository(); ··· 15 15 $path); 16 16 } 17 17 18 - protected function executeQueryFromFuture(Future $future) { 18 + protected function resolveFileContentFuture(Future $future) { 19 19 list($corpus) = $future->resolvex(); 20 - 21 - $file_content = new DiffusionFileContent(); 22 - $file_content->setCorpus($corpus); 23 - 24 - return $file_content; 20 + return $corpus; 25 21 } 26 22 27 23 }
+3 -7
src/applications/diffusion/query/filecontent/DiffusionMercurialFileContentQuery.php
··· 3 3 final class DiffusionMercurialFileContentQuery 4 4 extends DiffusionFileContentQuery { 5 5 6 - public function getFileContentFuture() { 6 + protected function getFileContentFuture() { 7 7 $drequest = $this->getRequest(); 8 8 9 9 $repository = $drequest->getRepository(); ··· 16 16 $path); 17 17 } 18 18 19 - protected function executeQueryFromFuture(Future $future) { 19 + protected function resolveFileContentFuture(Future $future) { 20 20 list($corpus) = $future->resolvex(); 21 - 22 - $file_content = new DiffusionFileContent(); 23 - $file_content->setCorpus($corpus); 24 - 25 - return $file_content; 21 + return $corpus; 26 22 } 27 23 28 24 }
+4 -25
src/applications/diffusion/query/filecontent/DiffusionSvnFileContentQuery.php
··· 2 2 3 3 final class DiffusionSvnFileContentQuery extends DiffusionFileContentQuery { 4 4 5 - public function getFileContentFuture() { 5 + protected function getFileContentFuture() { 6 6 $drequest = $this->getRequest(); 7 7 8 8 $repository = $drequest->getRepository(); ··· 14 14 $repository->getSubversionPathURI($path, $commit)); 15 15 } 16 16 17 - protected function executeQueryFromFuture(Future $future) { 18 - try { 19 - list($corpus) = $future->resolvex(); 20 - } catch (CommandException $ex) { 21 - $stderr = $ex->getStdErr(); 22 - if (preg_match('/path not found$/', trim($stderr))) { 23 - // TODO: Improve user experience for this. One way to end up here 24 - // is to have the parser behind and look at a file which was recently 25 - // nuked; Diffusion will think it still exists and try to grab content 26 - // at HEAD. 27 - throw new Exception( 28 - pht( 29 - 'Failed to retrieve file content from Subversion. The file may '. 30 - 'have been recently deleted, or the Diffusion cache may be out of '. 31 - 'date.')); 32 - } else { 33 - throw $ex; 34 - } 35 - } 36 - 37 - $file_content = new DiffusionFileContent(); 38 - $file_content->setCorpus($corpus); 39 - 40 - return $file_content; 17 + protected function resolveFileContentFuture(Future $future) { 18 + list($corpus) = $future->resolvex(); 19 + return $corpus; 41 20 } 42 21 43 22 }