@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Support CSV, JSON, and tab-separated text as export formats
Summary: Depends on D18919. Ref T13046. Adds some simple modular exporters.
Test Plan: Exported pull logs in each format.
Reviewers: amckinley
Reviewed By: amckinley
Maniphest Tasks: T13046
Differential Revision: https://secure.phabricator.com/D18934
+319
-37
+10
src/__phutil_library_map__.php
+10
src/__phutil_library_map__.php
···
2231
2231
'PhabricatorBulkEngine' => 'applications/transactions/bulk/PhabricatorBulkEngine.php',
2232
2232
'PhabricatorBulkManagementMakeSilentWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementMakeSilentWorkflow.php',
2233
2233
'PhabricatorBulkManagementWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementWorkflow.php',
2234
+
'PhabricatorCSVExportFormat' => 'infrastructure/export/PhabricatorCSVExportFormat.php',
2234
2235
'PhabricatorCacheDAO' => 'applications/cache/storage/PhabricatorCacheDAO.php',
2235
2236
'PhabricatorCacheEngine' => 'applications/system/engine/PhabricatorCacheEngine.php',
2236
2237
'PhabricatorCacheEngineExtension' => 'applications/system/engine/PhabricatorCacheEngineExtension.php',
···
2844
2845
'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php',
2845
2846
'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php',
2846
2847
'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php',
2848
+
'PhabricatorExportFormat' => 'infrastructure/export/PhabricatorExportFormat.php',
2847
2849
'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php',
2848
2850
'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php',
2849
2851
'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php',
···
3087
3089
'PhabricatorInlineSummaryView' => 'infrastructure/diff/view/PhabricatorInlineSummaryView.php',
3088
3090
'PhabricatorInstructionsEditField' => 'applications/transactions/editfield/PhabricatorInstructionsEditField.php',
3089
3091
'PhabricatorIntConfigType' => 'applications/config/type/PhabricatorIntConfigType.php',
3092
+
'PhabricatorIntExportField' => 'infrastructure/export/PhabricatorIntExportField.php',
3090
3093
'PhabricatorInternalSetting' => 'applications/settings/setting/PhabricatorInternalSetting.php',
3091
3094
'PhabricatorInternationalizationManagementExtractWorkflow' => 'infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php',
3092
3095
'PhabricatorInternationalizationManagementWorkflow' => 'infrastructure/internationalization/management/PhabricatorInternationalizationManagementWorkflow.php',
···
3096
3099
'PhabricatorIteratorFileUploadSource' => 'applications/files/uploadsource/PhabricatorIteratorFileUploadSource.php',
3097
3100
'PhabricatorJIRAAuthProvider' => 'applications/auth/provider/PhabricatorJIRAAuthProvider.php',
3098
3101
'PhabricatorJSONConfigType' => 'applications/config/type/PhabricatorJSONConfigType.php',
3102
+
'PhabricatorJSONExportFormat' => 'infrastructure/export/PhabricatorJSONExportFormat.php',
3099
3103
'PhabricatorJavelinLinter' => 'infrastructure/lint/linter/PhabricatorJavelinLinter.php',
3100
3104
'PhabricatorJiraIssueHasObjectEdgeType' => 'applications/doorkeeper/edge/PhabricatorJiraIssueHasObjectEdgeType.php',
3101
3105
'PhabricatorJumpNavHandler' => 'applications/search/engine/PhabricatorJumpNavHandler.php',
···
4245
4249
'PhabricatorTextAreaEditField' => 'applications/transactions/editfield/PhabricatorTextAreaEditField.php',
4246
4250
'PhabricatorTextConfigType' => 'applications/config/type/PhabricatorTextConfigType.php',
4247
4251
'PhabricatorTextEditField' => 'applications/transactions/editfield/PhabricatorTextEditField.php',
4252
+
'PhabricatorTextExportFormat' => 'infrastructure/export/PhabricatorTextExportFormat.php',
4248
4253
'PhabricatorTextListConfigType' => 'applications/config/type/PhabricatorTextListConfigType.php',
4249
4254
'PhabricatorTime' => 'infrastructure/time/PhabricatorTime.php',
4250
4255
'PhabricatorTimeFormatSetting' => 'applications/settings/setting/PhabricatorTimeFormatSetting.php',
···
7564
7569
'PhabricatorBulkEngine' => 'Phobject',
7565
7570
'PhabricatorBulkManagementMakeSilentWorkflow' => 'PhabricatorBulkManagementWorkflow',
7566
7571
'PhabricatorBulkManagementWorkflow' => 'PhabricatorManagementWorkflow',
7572
+
'PhabricatorCSVExportFormat' => 'PhabricatorExportFormat',
7567
7573
'PhabricatorCacheDAO' => 'PhabricatorLiskDAO',
7568
7574
'PhabricatorCacheEngine' => 'Phobject',
7569
7575
'PhabricatorCacheEngineExtension' => 'Phobject',
···
8268
8274
'PhabricatorExampleEventListener' => 'PhabricatorEventListener',
8269
8275
'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource',
8270
8276
'PhabricatorExportField' => 'Phobject',
8277
+
'PhabricatorExportFormat' => 'Phobject',
8271
8278
'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions',
8272
8279
'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck',
8273
8280
'PhabricatorExternalAccount' => array(
···
8551
8558
'PhabricatorInlineSummaryView' => 'AphrontView',
8552
8559
'PhabricatorInstructionsEditField' => 'PhabricatorEditField',
8553
8560
'PhabricatorIntConfigType' => 'PhabricatorTextConfigType',
8561
+
'PhabricatorIntExportField' => 'PhabricatorExportField',
8554
8562
'PhabricatorInternalSetting' => 'PhabricatorSetting',
8555
8563
'PhabricatorInternationalizationManagementExtractWorkflow' => 'PhabricatorInternationalizationManagementWorkflow',
8556
8564
'PhabricatorInternationalizationManagementWorkflow' => 'PhabricatorManagementWorkflow',
···
8560
8568
'PhabricatorIteratorFileUploadSource' => 'PhabricatorFileUploadSource',
8561
8569
'PhabricatorJIRAAuthProvider' => 'PhabricatorOAuth1AuthProvider',
8562
8570
'PhabricatorJSONConfigType' => 'PhabricatorTextConfigType',
8571
+
'PhabricatorJSONExportFormat' => 'PhabricatorExportFormat',
8563
8572
'PhabricatorJavelinLinter' => 'ArcanistLinter',
8564
8573
'PhabricatorJiraIssueHasObjectEdgeType' => 'PhabricatorEdgeType',
8565
8574
'PhabricatorJumpNavHandler' => 'Phobject',
···
9922
9931
'PhabricatorTextAreaEditField' => 'PhabricatorEditField',
9923
9932
'PhabricatorTextConfigType' => 'PhabricatorConfigType',
9924
9933
'PhabricatorTextEditField' => 'PhabricatorEditField',
9934
+
'PhabricatorTextExportFormat' => 'PhabricatorExportFormat',
9925
9935
'PhabricatorTextListConfigType' => 'PhabricatorTextConfigType',
9926
9936
'PhabricatorTime' => 'Phobject',
9927
9937
'PhabricatorTimeFormatSetting' => 'PhabricatorSelectSetting',
+1
-1
src/applications/diffusion/query/DiffusionPullLogSearchEngine.php
+1
-1
src/applications/diffusion/query/DiffusionPullLogSearchEngine.php
+71
-34
src/applications/search/controller/PhabricatorApplicationSearchController.php
+71
-34
src/applications/search/controller/PhabricatorApplicationSearchController.php
···
413
413
$filename = phutil_utf8_strtolower($filename);
414
414
$filename = PhabricatorFile::normalizeFileName($filename);
415
415
416
+
$formats = PhabricatorExportFormat::getAllEnabledExportFormats();
417
+
$format_options = mpull($formats, 'getExportFormatName');
418
+
419
+
$errors = array();
420
+
421
+
$e_format = null;
416
422
if ($request->isFormPost()) {
417
-
$query = $engine->buildQueryFromSavedQuery($saved_query);
423
+
$format_key = $request->getStr('format');
424
+
$format = idx($formats, $format_key);
418
425
419
-
// NOTE: We aren't reading the pager from the request. Exports always
420
-
// affect the entire result set.
421
-
$pager = $engine->newPagerForSavedQuery($saved_query);
422
-
$pager->setPageSize(0x7FFFFFFF);
426
+
if (!$format) {
427
+
$e_format = pht('Invalid');
428
+
$errors[] = pht('Choose a valid export format.');
429
+
}
430
+
431
+
if (!$errors) {
432
+
$query = $engine->buildQueryFromSavedQuery($saved_query);
423
433
424
-
$objects = $engine->executeQuery($query, $pager);
434
+
// NOTE: We aren't reading the pager from the request. Exports always
435
+
// affect the entire result set.
436
+
$pager = $engine->newPagerForSavedQuery($saved_query);
437
+
$pager->setPageSize(0x7FFFFFFF);
425
438
426
-
$extension = 'json';
427
-
$mime_type = 'application/json';
428
-
$filename = $filename.'.'.$extension;
439
+
$objects = $engine->executeQuery($query, $pager);
440
+
441
+
$extension = $format->getFileExtension();
442
+
$mime_type = $format->getMIMEContentType();
443
+
$filename = $filename.'.'.$extension;
444
+
445
+
$format = clone $format;
446
+
$format->setViewer($viewer);
447
+
448
+
$export_data = $engine->newExport($objects);
449
+
450
+
if (count($export_data) !== count($objects)) {
451
+
throw new Exception(
452
+
pht(
453
+
'Search engine exported the wrong number of objects, expected '.
454
+
'%s but got %s.',
455
+
phutil_count($objects),
456
+
phutil_count($export_data)));
457
+
}
458
+
459
+
$objects = array_values($objects);
460
+
$export_data = array_values($export_data);
461
+
462
+
$field_list = $engine->newExportFieldList();
463
+
$field_list = mpull($field_list, null, 'getKey');
464
+
465
+
for ($ii = 0; $ii < count($objects); $ii++) {
466
+
$format->addObject($objects[$ii], $field_list, $export_data[$ii]);
467
+
}
429
468
430
-
$result = $engine->newExport($objects);
431
-
$result = id(new PhutilJSON())
432
-
->encodeAsList($result);
469
+
$export_result = $format->newFileData();
433
470
434
-
$file = PhabricatorFile::newFromFileData(
435
-
$result,
436
-
array(
437
-
'name' => $filename,
438
-
'authorPHID' => $viewer->getPHID(),
439
-
'ttl.relative' => phutil_units('15 minutes in seconds'),
440
-
'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
441
-
'mime-type' => $mime_type,
442
-
));
471
+
$file = PhabricatorFile::newFromFileData(
472
+
$export_result,
473
+
array(
474
+
'name' => $filename,
475
+
'authorPHID' => $viewer->getPHID(),
476
+
'ttl.relative' => phutil_units('15 minutes in seconds'),
477
+
'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
478
+
'mime-type' => $mime_type,
479
+
));
443
480
444
-
return $this->newDialog()
445
-
->setTitle(pht('Download Results'))
446
-
->appendParagraph(
447
-
pht('Click the download button to download the exported data.'))
448
-
->addCancelButton($cancel_uri, pht('Done'))
449
-
->setSubmitURI($file->getDownloadURI())
450
-
->setDisableWorkflowOnSubmit(true)
451
-
->addSubmitButton(pht('Download Results'));
481
+
return $this->newDialog()
482
+
->setTitle(pht('Download Results'))
483
+
->appendParagraph(
484
+
pht('Click the download button to download the exported data.'))
485
+
->addCancelButton($cancel_uri, pht('Done'))
486
+
->setSubmitURI($file->getDownloadURI())
487
+
->setDisableWorkflowOnSubmit(true)
488
+
->addSubmitButton(pht('Download Data'));
489
+
}
452
490
}
453
491
454
492
$export_form = id(new AphrontFormView())
···
457
495
id(new AphrontFormSelectControl())
458
496
->setName('format')
459
497
->setLabel(pht('Format'))
460
-
->setOptions(
461
-
array(
462
-
'json' => 'JSON',
463
-
)));
498
+
->setError($e_format)
499
+
->setOptions($format_options));
464
500
465
501
return $this->newDialog()
466
502
->setTitle(pht('Export Results'))
503
+
->setErrors($errors)
467
504
->appendForm($export_form)
468
505
->addCancelButton($cancel_uri)
469
506
->addSubmitButton(pht('Continue'));
···
826
863
$export_uri = $engine->getExportURI($query_key);
827
864
$actions[] = id(new PhabricatorActionView())
828
865
->setIcon('fa-download')
829
-
->setName(pht('Export Results'))
866
+
->setName(pht('Export Data'))
830
867
->setWorkflow(true)
831
868
->setHref($export_uri);
832
869
}
+4
src/applications/search/engine/PhabricatorApplicationSearchEngine.php
+4
src/applications/search/engine/PhabricatorApplicationSearchEngine.php
+47
src/infrastructure/export/PhabricatorCSVExportFormat.php
+47
src/infrastructure/export/PhabricatorCSVExportFormat.php
···
1
+
<?php
2
+
3
+
final class PhabricatorCSVExportFormat
4
+
extends PhabricatorExportFormat {
5
+
6
+
const EXPORTKEY = 'csv';
7
+
8
+
private $rows = array();
9
+
10
+
public function getExportFormatName() {
11
+
return pht('Comma-Separated Values (.csv)');
12
+
}
13
+
14
+
public function isExportFormatEnabled() {
15
+
return true;
16
+
}
17
+
18
+
public function getFileExtension() {
19
+
return 'csv';
20
+
}
21
+
22
+
public function getMIMEContentType() {
23
+
return 'text/csv';
24
+
}
25
+
26
+
public function addObject($object, array $fields, array $map) {
27
+
$values = array();
28
+
foreach ($fields as $key => $field) {
29
+
$value = $map[$key];
30
+
$value = $field->getTextValue($value);
31
+
32
+
if (preg_match('/\s|,|\"/', $value)) {
33
+
$value = str_replace('"', '""', $value);
34
+
$value = '"'.$value.'"';
35
+
}
36
+
37
+
$values[] = $value;
38
+
}
39
+
40
+
$this->rows[] = implode(',', $values);
41
+
}
42
+
43
+
public function newFileData() {
44
+
return implode("\n", $this->rows);
45
+
}
46
+
47
+
}
+24
-1
src/infrastructure/export/PhabricatorEpochExportField.php
+24
-1
src/infrastructure/export/PhabricatorEpochExportField.php
···
1
1
<?php
2
2
3
3
final class PhabricatorEpochExportField
4
-
extends PhabricatorExportField {}
4
+
extends PhabricatorExportField {
5
+
6
+
private $zone;
7
+
8
+
public function getTextValue($value) {
9
+
if (!isset($this->zone)) {
10
+
$this->zone = new DateTimeZone('UTC');
11
+
}
12
+
13
+
try {
14
+
$date = new DateTime('@'.$value);
15
+
} catch (Exception $ex) {
16
+
return null;
17
+
}
18
+
19
+
$date->setTimezone($this->zone);
20
+
return $date->format('c');
21
+
}
22
+
23
+
public function getNaturalValue($value) {
24
+
return (int)$value;
25
+
}
26
+
27
+
}
+8
src/infrastructure/export/PhabricatorExportField.php
+8
src/infrastructure/export/PhabricatorExportField.php
+51
src/infrastructure/export/PhabricatorExportFormat.php
+51
src/infrastructure/export/PhabricatorExportFormat.php
···
1
+
<?php
2
+
3
+
abstract class PhabricatorExportFormat
4
+
extends Phobject {
5
+
6
+
private $viewer;
7
+
8
+
final public function getExportFormatKey() {
9
+
return $this->getPhobjectClassConstant('EXPORTKEY');
10
+
}
11
+
12
+
final public function setViewer(PhabricatorUser $viewer) {
13
+
$this->viewer = $viewer;
14
+
return $this;
15
+
}
16
+
17
+
final public function getViewer() {
18
+
return $this->viewer;
19
+
}
20
+
21
+
abstract public function getExportFormatName();
22
+
abstract public function getMIMEContentType();
23
+
abstract public function getFileExtension();
24
+
25
+
abstract public function addObject($object, array $fields, array $map);
26
+
abstract public function newFileData();
27
+
28
+
public function isExportFormatEnabled() {
29
+
return true;
30
+
}
31
+
32
+
final public static function getAllExportFormats() {
33
+
return id(new PhutilClassMapQuery())
34
+
->setAncestorClass(__CLASS__)
35
+
->setUniqueMethod('getExportFormatKey')
36
+
->execute();
37
+
}
38
+
39
+
final public static function getAllEnabledExportFormats() {
40
+
$formats = self::getAllExportFormats();
41
+
42
+
foreach ($formats as $key => $format) {
43
+
if (!$format->isExportFormatEnabled()) {
44
+
unset($formats[$key]);
45
+
}
46
+
}
47
+
48
+
return $formats;
49
+
}
50
+
51
+
}
+7
-1
src/infrastructure/export/PhabricatorIDExportField.php
+7
-1
src/infrastructure/export/PhabricatorIDExportField.php
+10
src/infrastructure/export/PhabricatorIntExportField.php
+10
src/infrastructure/export/PhabricatorIntExportField.php
+43
src/infrastructure/export/PhabricatorJSONExportFormat.php
+43
src/infrastructure/export/PhabricatorJSONExportFormat.php
···
1
+
<?php
2
+
3
+
final class PhabricatorJSONExportFormat
4
+
extends PhabricatorExportFormat {
5
+
6
+
const EXPORTKEY = 'json';
7
+
8
+
private $objects = array();
9
+
10
+
public function getExportFormatName() {
11
+
return 'JSON (.json)';
12
+
}
13
+
14
+
public function isExportFormatEnabled() {
15
+
return true;
16
+
}
17
+
18
+
public function getFileExtension() {
19
+
return 'json';
20
+
}
21
+
22
+
public function getMIMEContentType() {
23
+
return 'application/json';
24
+
}
25
+
26
+
public function addObject($object, array $fields, array $map) {
27
+
$values = array();
28
+
foreach ($fields as $key => $field) {
29
+
$value = $map[$key];
30
+
$value = $field->getNaturalValue($value);
31
+
32
+
$values[$key] = $value;
33
+
}
34
+
35
+
$this->objects[] = $values;
36
+
}
37
+
38
+
public function newFileData() {
39
+
return id(new PhutilJSON())
40
+
->encodeAsList($this->objects);
41
+
}
42
+
43
+
}
+43
src/infrastructure/export/PhabricatorTextExportFormat.php
+43
src/infrastructure/export/PhabricatorTextExportFormat.php
···
1
+
<?php
2
+
3
+
final class PhabricatorTextExportFormat
4
+
extends PhabricatorExportFormat {
5
+
6
+
const EXPORTKEY = 'text';
7
+
8
+
private $rows = array();
9
+
10
+
public function getExportFormatName() {
11
+
return 'Tab-Separated Text (.txt)';
12
+
}
13
+
14
+
public function isExportFormatEnabled() {
15
+
return true;
16
+
}
17
+
18
+
public function getFileExtension() {
19
+
return 'txt';
20
+
}
21
+
22
+
public function getMIMEContentType() {
23
+
return 'text/plain';
24
+
}
25
+
26
+
public function addObject($object, array $fields, array $map) {
27
+
$values = array();
28
+
foreach ($fields as $key => $field) {
29
+
$value = $map[$key];
30
+
$value = $field->getTextValue($value);
31
+
$value = addcslashes($value, "\0..\37\\\177..\377");
32
+
33
+
$values[] = $value;
34
+
}
35
+
36
+
$this->rows[] = implode("\t", $values);
37
+
}
38
+
39
+
public function newFileData() {
40
+
return implode("\n", $this->rows)."\n";
41
+
}
42
+
43
+
}