recaptime.dev / phorge
@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.) hq.recaptime.dev/wiki/Phorge
phorge phabricator
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Add a basic, general-purpose export workflow for all objects with SearchEngine support

Summary:
Depends on D18918. Ref T13046. Ref T5954. Pull logs can currently be browsed in the web UI, but this isn't very powerful, especially if you have thousands of them.

Allow SearchEngine implementations to define exportable fields so that users can "Use Results > Export Data" on any query. In particular, they can use this workflow to download a file with pull logs.

In the future, this can replace the existing "Export to Excel" feature in Maniphest.

For now, we hard-code JSON as the only supported datatype and don't actually make any effort to format the data properly, but this leaves room to add more exporters (CSV, Excel) and data type awareness (integer casting, date formatting, etc) in the future.

For sufficiently large result sets, this will probably time out. At some point, I'll make this use the job queue (like bulk editing) when the export is "large" (affects more than 1K rows?).

Test Plan: Downloaded pull logs in JSON format.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13046, T5954

Differential Revision: https://secure.phabricator.com/D18919

epriestley c0b8e478 5058cfb9

+274 -2
+10
src/__phutil_library_map__.php
··· 2836 2836 'PhabricatorEnv' => 'infrastructure/env/PhabricatorEnv.php', 2837 2837 'PhabricatorEnvTestCase' => 'infrastructure/env/__tests__/PhabricatorEnvTestCase.php', 2838 2838 'PhabricatorEpochEditField' => 'applications/transactions/editfield/PhabricatorEpochEditField.php', 2839 + 'PhabricatorEpochExportField' => 'infrastructure/export/PhabricatorEpochExportField.php', 2839 2840 'PhabricatorEvent' => 'infrastructure/events/PhabricatorEvent.php', 2840 2841 'PhabricatorEventEngine' => 'infrastructure/events/PhabricatorEventEngine.php', 2841 2842 'PhabricatorEventListener' => 'infrastructure/events/PhabricatorEventListener.php', 2842 2843 'PhabricatorEventType' => 'infrastructure/events/constant/PhabricatorEventType.php', 2843 2844 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', 2844 2845 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', 2846 + 'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php', 2845 2847 'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php', 2846 2848 'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php', 2847 2849 'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php', ··· 3061 3063 'PhabricatorHomeProfileMenuItem' => 'applications/home/menuitem/PhabricatorHomeProfileMenuItem.php', 3062 3064 'PhabricatorHovercardEngineExtension' => 'applications/search/engineextension/PhabricatorHovercardEngineExtension.php', 3063 3065 'PhabricatorHovercardEngineExtensionModule' => 'applications/search/engineextension/PhabricatorHovercardEngineExtensionModule.php', 3066 + 'PhabricatorIDExportField' => 'infrastructure/export/PhabricatorIDExportField.php', 3064 3067 'PhabricatorIDsSearchEngineExtension' => 'applications/search/engineextension/PhabricatorIDsSearchEngineExtension.php', 3065 3068 'PhabricatorIDsSearchField' => 'applications/search/field/PhabricatorIDsSearchField.php', 3066 3069 'PhabricatorIconDatasource' => 'applications/files/typeahead/PhabricatorIconDatasource.php', ··· 3412 3415 'PhabricatorPHDConfigOptions' => 'applications/config/option/PhabricatorPHDConfigOptions.php', 3413 3416 'PhabricatorPHID' => 'applications/phid/storage/PhabricatorPHID.php', 3414 3417 'PhabricatorPHIDConstants' => 'applications/phid/PhabricatorPHIDConstants.php', 3418 + 'PhabricatorPHIDExportField' => 'infrastructure/export/PhabricatorPHIDExportField.php', 3415 3419 'PhabricatorPHIDInterface' => 'applications/phid/interface/PhabricatorPHIDInterface.php', 3416 3420 'PhabricatorPHIDListEditField' => 'applications/transactions/editfield/PhabricatorPHIDListEditField.php', 3417 3421 'PhabricatorPHIDListEditType' => 'applications/transactions/edittype/PhabricatorPHIDListEditType.php', ··· 4177 4181 'PhabricatorStorageSchemaSpec' => 'infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php', 4178 4182 'PhabricatorStorageSetupCheck' => 'applications/config/check/PhabricatorStorageSetupCheck.php', 4179 4183 'PhabricatorStringConfigType' => 'applications/config/type/PhabricatorStringConfigType.php', 4184 + 'PhabricatorStringExportField' => 'infrastructure/export/PhabricatorStringExportField.php', 4180 4185 'PhabricatorStringListConfigType' => 'applications/config/type/PhabricatorStringListConfigType.php', 4181 4186 'PhabricatorStringListEditField' => 'applications/transactions/editfield/PhabricatorStringListEditField.php', 4182 4187 'PhabricatorStringSetting' => 'applications/settings/setting/PhabricatorStringSetting.php', ··· 8255 8260 'PhabricatorEnv' => 'Phobject', 8256 8261 'PhabricatorEnvTestCase' => 'PhabricatorTestCase', 8257 8262 'PhabricatorEpochEditField' => 'PhabricatorEditField', 8263 + 'PhabricatorEpochExportField' => 'PhabricatorExportField', 8258 8264 'PhabricatorEvent' => 'PhutilEvent', 8259 8265 'PhabricatorEventEngine' => 'Phobject', 8260 8266 'PhabricatorEventListener' => 'PhutilEventListener', 8261 8267 'PhabricatorEventType' => 'PhutilEventType', 8262 8268 'PhabricatorExampleEventListener' => 'PhabricatorEventListener', 8263 8269 'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource', 8270 + 'PhabricatorExportField' => 'Phobject', 8264 8271 'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions', 8265 8272 'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck', 8266 8273 'PhabricatorExternalAccount' => array( ··· 8521 8528 'PhabricatorHomeProfileMenuItem' => 'PhabricatorProfileMenuItem', 8522 8529 'PhabricatorHovercardEngineExtension' => 'Phobject', 8523 8530 'PhabricatorHovercardEngineExtensionModule' => 'PhabricatorConfigModule', 8531 + 'PhabricatorIDExportField' => 'PhabricatorExportField', 8524 8532 'PhabricatorIDsSearchEngineExtension' => 'PhabricatorSearchEngineExtension', 8525 8533 'PhabricatorIDsSearchField' => 'PhabricatorSearchField', 8526 8534 'PhabricatorIconDatasource' => 'PhabricatorTypeaheadDatasource', ··· 8911 8919 'PhabricatorPHDConfigOptions' => 'PhabricatorApplicationConfigOptions', 8912 8920 'PhabricatorPHID' => 'Phobject', 8913 8921 'PhabricatorPHIDConstants' => 'Phobject', 8922 + 'PhabricatorPHIDExportField' => 'PhabricatorExportField', 8914 8923 'PhabricatorPHIDListEditField' => 'PhabricatorEditField', 8915 8924 'PhabricatorPHIDListEditType' => 'PhabricatorEditType', 8916 8925 'PhabricatorPHIDResolver' => 'Phobject', ··· 9850 9859 'PhabricatorStorageSchemaSpec' => 'PhabricatorConfigSchemaSpec', 9851 9860 'PhabricatorStorageSetupCheck' => 'PhabricatorSetupCheck', 9852 9861 'PhabricatorStringConfigType' => 'PhabricatorTextConfigType', 9862 + 'PhabricatorStringExportField' => 'PhabricatorExportField', 9853 9863 'PhabricatorStringListConfigType' => 'PhabricatorTextListConfigType', 9854 9864 'PhabricatorStringListEditField' => 'PhabricatorEditField', 9855 9865 'PhabricatorStringSetting' => 'PhabricatorSetting',
+1 -1
src/applications/base/PhabricatorApplication.php
··· 623 623 } 624 624 625 625 protected function getQueryRoutePattern($base = null) { 626 - return $base.'(?:query/(?P<queryKey>[^/]+)/)?'; 626 + return $base.'(?:query/(?P<queryKey>[^/]+)/(?:(?P<queryAction>[^/]+)/))?'; 627 627 } 628 628 629 629 protected function getProfileMenuRouting($controller) {
+81
src/applications/diffusion/query/DiffusionPullLogSearchEngine.php
··· 47 47 ); 48 48 } 49 49 50 + protected function newExportFields() { 51 + return array( 52 + id(new PhabricatorIDExportField()) 53 + ->setKey('id') 54 + ->setLabel(pht('ID')), 55 + id(new PhabricatorPHIDExportField()) 56 + ->setKey('phid') 57 + ->setLabel(pht('PHID')), 58 + id(new PhabricatorPHIDExportField()) 59 + ->setKey('repositoryPHID') 60 + ->setLabel(pht('Repository PHID')), 61 + id(new PhabricatorStringExportField()) 62 + ->setKey('repository') 63 + ->setLabel(pht('Repository')), 64 + id(new PhabricatorPHIDExportField()) 65 + ->setKey('pullerPHID') 66 + ->setLabel(pht('Puller PHID')), 67 + id(new PhabricatorStringExportField()) 68 + ->setKey('puller') 69 + ->setLabel(pht('Puller')), 70 + id(new PhabricatorStringExportField()) 71 + ->setKey('protocol') 72 + ->setLabel(pht('Protocol')), 73 + id(new PhabricatorStringExportField()) 74 + ->setKey('result') 75 + ->setLabel(pht('Result')), 76 + id(new PhabricatorStringExportField()) 77 + ->setKey('code') 78 + ->setLabel(pht('Code')), 79 + id(new PhabricatorEpochExportField()) 80 + ->setKey('date') 81 + ->setLabel(pht('Date')), 82 + ); 83 + } 84 + 85 + public function newExport(array $events) { 86 + $viewer = $this->requireViewer(); 87 + 88 + $phids = array(); 89 + foreach ($events as $event) { 90 + if ($event->getPullerPHID()) { 91 + $phids[] = $event->getPullerPHID(); 92 + } 93 + } 94 + $handles = $viewer->loadHandles($phids); 95 + 96 + $export = array(); 97 + foreach ($events as $event) { 98 + $repository = $event->getRepository(); 99 + if ($repository) { 100 + $repository_phid = $repository->getPHID(); 101 + $repository_name = $repository->getDisplayName(); 102 + } else { 103 + $repository_phid = null; 104 + $repository_name = null; 105 + } 106 + 107 + $puller_phid = $event->getPullerPHID(); 108 + if ($puller_phid) { 109 + $puller_name = $handles[$puller_phid]->getName(); 110 + } else { 111 + $puller_name = null; 112 + } 113 + 114 + $export[] = array( 115 + 'id' => $event->getID(), 116 + 'phid' => $event->getPHID(), 117 + 'repositoryPHID' => $repository_phid, 118 + 'repository' => $repository_name, 119 + 'pullerPHID' => $puller_phid, 120 + 'puller' => $puller_name, 121 + 'protocol' => $event->getRemoteProtocol(), 122 + 'result' => $event->getResultType(), 123 + 'code' => $event->getResultCode(), 124 + 'date' => $event->getEpoch(), 125 + ); 126 + } 127 + 128 + return $export; 129 + } 130 + 50 131 protected function getURI($path) { 51 132 return '/diffusion/pulllog/'.$path; 52 133 }
+122 -1
src/applications/search/controller/PhabricatorApplicationSearchController.php
··· 66 66 public function processRequest() { 67 67 $this->validateDelegatingController(); 68 68 69 + $query_action = $this->getRequest()->getURIData('queryAction'); 70 + if ($query_action == 'export') { 71 + return $this->processExportRequest(); 72 + } 73 + 69 74 $key = $this->getQueryKey(); 70 75 if ($key == 'edit') { 71 76 return $this->processEditRequest(); ··· 374 379 ->appendChild($body); 375 380 } 376 381 382 + private function processExportRequest() { 383 + $viewer = $this->getViewer(); 384 + $engine = $this->getSearchEngine(); 385 + $request = $this->getRequest(); 386 + 387 + if (!$this->canExport()) { 388 + return new Aphront404Response(); 389 + } 390 + 391 + $query_key = $this->getQueryKey(); 392 + if ($engine->isBuiltinQuery($query_key)) { 393 + $saved_query = $engine->buildSavedQueryFromBuiltin($query_key); 394 + } else if ($query_key) { 395 + $saved_query = id(new PhabricatorSavedQueryQuery()) 396 + ->setViewer($viewer) 397 + ->withQueryKeys(array($query_key)) 398 + ->executeOne(); 399 + if (!$saved_query) { 400 + return new Aphront404Response(); 401 + } 402 + } 403 + 404 + $cancel_uri = $engine->getQueryResultsPageURI($query_key); 405 + 406 + $named_query = idx($engine->loadEnabledNamedQueries(), $query_key); 407 + 408 + if ($named_query) { 409 + $filename = $named_query->getQueryName(); 410 + } else { 411 + $filename = $engine->getResultTypeDescription(); 412 + } 413 + $filename = phutil_utf8_strtolower($filename); 414 + $filename = PhabricatorFile::normalizeFileName($filename); 415 + 416 + if ($request->isFormPost()) { 417 + $query = $engine->buildQueryFromSavedQuery($saved_query); 418 + 419 + // NOTE: We aren't reading the pager from the request. Exports always 420 + // affect the entire result set. 421 + $pager = $engine->newPagerForSavedQuery($saved_query); 422 + $pager->setPageSize(0x7FFFFFFF); 423 + 424 + $objects = $engine->executeQuery($query, $pager); 425 + 426 + $extension = 'json'; 427 + $mime_type = 'application/json'; 428 + $filename = $filename.'.'.$extension; 429 + 430 + $result = $engine->newExport($objects); 431 + $result = id(new PhutilJSON()) 432 + ->encodeAsList($result); 433 + 434 + $file = PhabricatorFile::newFromFileData( 435 + $result, 436 + array( 437 + 'name' => $filename, 438 + 'authorPHID' => $viewer->getPHID(), 439 + 'ttl.relative' => phutil_units('15 minutes in seconds'), 440 + 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, 441 + 'mime-type' => $mime_type, 442 + )); 443 + 444 + return $this->newDialog() 445 + ->setTitle(pht('Download Results')) 446 + ->appendParagraph( 447 + pht('Click the download button to download the exported data.')) 448 + ->addCancelButton($cancel_uri, pht('Done')) 449 + ->setSubmitURI($file->getDownloadURI()) 450 + ->setDisableWorkflowOnSubmit(true) 451 + ->addSubmitButton(pht('Download Results')); 452 + } 453 + 454 + $export_form = id(new AphrontFormView()) 455 + ->setViewer($viewer) 456 + ->appendControl( 457 + id(new AphrontFormSelectControl()) 458 + ->setName('format') 459 + ->setLabel(pht('Format')) 460 + ->setOptions( 461 + array( 462 + 'json' => 'JSON', 463 + ))); 464 + 465 + return $this->newDialog() 466 + ->setTitle(pht('Export Results')) 467 + ->appendForm($export_form) 468 + ->addCancelButton($cancel_uri) 469 + ->addSubmitButton(pht('Continue')); 470 + } 471 + 377 472 private function processEditRequest() { 378 473 $parent = $this->getDelegatingController(); 379 474 $request = $this->getRequest(); ··· 720 815 $viewer); 721 816 722 817 if ($can_use && $is_installed) { 723 - $dashboard_uri = '/dashboard/install/'; 724 818 $actions[] = id(new PhabricatorActionView()) 725 819 ->setIcon('fa-dashboard') 726 820 ->setName(pht('Add to Dashboard')) 727 821 ->setWorkflow(true) 728 822 ->setHref("/dashboard/panel/install/{$engine_class}/{$query_key}/"); 823 + } 824 + 825 + if ($this->canExport()) { 826 + $export_uri = $engine->getExportURI($query_key); 827 + $actions[] = id(new PhabricatorActionView()) 828 + ->setIcon('fa-download') 829 + ->setName(pht('Export Results')) 830 + ->setWorkflow(true) 831 + ->setHref($export_uri); 729 832 } 730 833 731 834 if ($is_dev) { ··· 751 854 } 752 855 753 856 return $actions; 857 + } 858 + 859 + private function canExport() { 860 + $engine = $this->getSearchEngine(); 861 + if (!$engine->canExport()) { 862 + return false; 863 + } 864 + 865 + // Don't allow logged-out users to perform exports. There's no technical 866 + // or policy reason they can't, but we don't normally give them access 867 + // to write files or jobs. For now, just err on the side of caution. 868 + 869 + $viewer = $this->getViewer(); 870 + if (!$viewer->getPHID()) { 871 + return false; 872 + } 873 + 874 + return true; 754 875 } 755 876 756 877 }
+17
src/applications/search/engine/PhabricatorApplicationSearchEngine.php
··· 413 413 return $this->getURI(''); 414 414 } 415 415 416 + public function getExportURI($query_key) { 417 + return $this->getURI('query/'.$query_key.'/export/'); 418 + } 419 + 416 420 417 421 /** 418 422 * Return the URI to a path within the application. Used to construct default ··· 1438 1442 } 1439 1443 1440 1444 public function newUseResultsActions(PhabricatorSavedQuery $saved) { 1445 + return array(); 1446 + } 1447 + 1448 + 1449 + /* -( Export )------------------------------------------------------------- */ 1450 + 1451 + 1452 + public function canExport() { 1453 + $fields = $this->newExportFields(); 1454 + return (bool)$fields; 1455 + } 1456 + 1457 + protected function newExportFields() { 1441 1458 return array(); 1442 1459 } 1443 1460
+4
src/infrastructure/export/PhabricatorEpochExportField.php
··· 1 + <?php 2 + 3 + final class PhabricatorEpochExportField 4 + extends PhabricatorExportField {}
+27
src/infrastructure/export/PhabricatorExportField.php
··· 1 + <?php 2 + 3 + abstract class PhabricatorExportField 4 + extends Phobject { 5 + 6 + private $key; 7 + private $label; 8 + 9 + public function setKey($key) { 10 + $this->key = $key; 11 + return $this; 12 + } 13 + 14 + public function getKey() { 15 + return $this->key; 16 + } 17 + 18 + public function setLabel($label) { 19 + $this->label = $label; 20 + return $this; 21 + } 22 + 23 + public function getLabel() { 24 + return $this->label; 25 + } 26 + 27 + }
+4
src/infrastructure/export/PhabricatorIDExportField.php
··· 1 + <?php 2 + 3 + final class PhabricatorIDExportField 4 + extends PhabricatorExportField {}
+4
src/infrastructure/export/PhabricatorPHIDExportField.php
··· 1 + <?php 2 + 3 + final class PhabricatorPHIDExportField 4 + extends PhabricatorExportField {}
+4
src/infrastructure/export/PhabricatorStringExportField.php
··· 1 + <?php 2 + 3 + final class PhabricatorStringExportField 4 + extends PhabricatorExportField {}