@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a `bin/bulk export` CLI tool to make debugging and profiling large exports easier
Summary:
Ref T13049. When stuff executes asynchronously on the bulk workflow it can be hard to inspect directly, and/or a pain to test because you have to go through a bunch of steps to run it again.
Make future work here easier by making export triggerable from the CLI. This makes it easy to repeat, inspect with `--trace`, profile with `--xprofile`, etc.
Test Plan:
- Ran several invalid commands, got sensible error messages.
- Ran some valid commands, got exported data.
- Used `--xprofile` to look at the profile for a 300MB dump of 100K tasks which took about 40 seconds to export. Nothing jumped out as sketchy to me -- CustomField wrangling is a little slow but most of the time looked like it was being spent legitimately.
Reviewers: amckinley
Reviewed By: amckinley
Maniphest Tasks: T13049
Differential Revision: https://secure.phabricator.com/D18965
+170
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
2226
2226
'PhabricatorBulkContentSource' => 'infrastructure/daemon/contentsource/PhabricatorBulkContentSource.php',
2227
2227
'PhabricatorBulkEditGroup' => 'applications/transactions/bulk/PhabricatorBulkEditGroup.php',
2228
2228
'PhabricatorBulkEngine' => 'applications/transactions/bulk/PhabricatorBulkEngine.php',
2229
+
'PhabricatorBulkManagementExportWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php',
2229
2230
'PhabricatorBulkManagementMakeSilentWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementMakeSilentWorkflow.php',
2230
2231
'PhabricatorBulkManagementWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementWorkflow.php',
2231
2232
'PhabricatorCSVExportFormat' => 'infrastructure/export/format/PhabricatorCSVExportFormat.php',
···
7579
7580
'PhabricatorBulkContentSource' => 'PhabricatorContentSource',
7580
7581
'PhabricatorBulkEditGroup' => 'Phobject',
7581
7582
'PhabricatorBulkEngine' => 'Phobject',
7583
+
'PhabricatorBulkManagementExportWorkflow' => 'PhabricatorBulkManagementWorkflow',
7582
7584
'PhabricatorBulkManagementMakeSilentWorkflow' => 'PhabricatorBulkManagementWorkflow',
7583
7585
'PhabricatorBulkManagementWorkflow' => 'PhabricatorManagementWorkflow',
7584
7586
'PhabricatorCSVExportFormat' => 'PhabricatorExportFormat',
+168
src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php
+168
src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhabricatorBulkManagementExportWorkflow
4
+
extends PhabricatorBulkManagementWorkflow {
5
+
6
+
protected function didConstruct() {
7
+
$this
8
+
->setName('export')
9
+
->setExamples('**export** [options]')
10
+
->setSynopsis(
11
+
pht('Export data to a flat file (JSON, CSV, Excel, etc).'))
12
+
->setArguments(
13
+
array(
14
+
array(
15
+
'name' => 'class',
16
+
'param' => 'class',
17
+
'help' => pht(
18
+
'SearchEngine class to export data from.'),
19
+
),
20
+
array(
21
+
'name' => 'format',
22
+
'param' => 'format',
23
+
'help' => pht('Export format.'),
24
+
),
25
+
array(
26
+
'name' => 'query',
27
+
'param' => 'key',
28
+
'help' => pht(
29
+
'Export the data selected by this query.'),
30
+
),
31
+
array(
32
+
'name' => 'output',
33
+
'param' => 'path',
34
+
'help' => pht(
35
+
'Write output to a file. If omitted, output will be sent to '.
36
+
'stdout.'),
37
+
),
38
+
array(
39
+
'name' => 'overwrite',
40
+
'help' => pht(
41
+
'If the output file already exists, overwrite it instead of '.
42
+
'raising an error.'),
43
+
),
44
+
));
45
+
}
46
+
47
+
public function execute(PhutilArgumentParser $args) {
48
+
$viewer = $this->getViewer();
49
+
50
+
$class = $args->getArg('class');
51
+
52
+
if (!strlen($class)) {
53
+
throw new PhutilArgumentUsageException(
54
+
pht(
55
+
'Specify a search engine class to export data from with '.
56
+
'"--class".'));
57
+
}
58
+
59
+
if (!is_subclass_of($class, 'PhabricatorApplicationSearchEngine')) {
60
+
throw new PhutilArgumentUsageException(
61
+
pht(
62
+
'SearchEngine class ("%s") is unknown.',
63
+
$class));
64
+
}
65
+
66
+
$engine = newv($class, array())
67
+
->setViewer($viewer);
68
+
69
+
if (!$engine->canExport()) {
70
+
throw new PhutilArgumentUsageException(
71
+
pht(
72
+
'SearchEngine class ("%s") does not support data export.',
73
+
$class));
74
+
}
75
+
76
+
$query_key = $args->getArg('query');
77
+
if (!strlen($query_key)) {
78
+
throw new PhutilArgumentUsageException(
79
+
pht(
80
+
'Specify a query to export with "--query".'));
81
+
}
82
+
83
+
if ($engine->isBuiltinQuery($query_key)) {
84
+
$saved_query = $engine->buildSavedQueryFromBuiltin($query_key);
85
+
} else if ($query_key) {
86
+
$saved_query = id(new PhabricatorSavedQueryQuery())
87
+
->setViewer($viewer)
88
+
->withQueryKeys(array($query_key))
89
+
->executeOne();
90
+
} else {
91
+
$saved_query = null;
92
+
}
93
+
94
+
if (!$saved_query) {
95
+
throw new PhutilArgumentUsageException(
96
+
pht(
97
+
'Failed to load saved query ("%s").',
98
+
$query_key));
99
+
}
100
+
101
+
$format_key = $args->getArg('format');
102
+
if (!strlen($format_key)) {
103
+
throw new PhutilArgumentUsageException(
104
+
pht(
105
+
'Specify an export format with "--format".'));
106
+
}
107
+
108
+
$all_formats = PhabricatorExportFormat::getAllExportFormats();
109
+
$format = idx($all_formats, $format_key);
110
+
if (!$format) {
111
+
throw new PhutilArgumentUsageException(
112
+
pht(
113
+
'Unknown export format ("%s"). Known formats are: %s.',
114
+
$format_key,
115
+
implode(', ', array_keys($all_formats))));
116
+
}
117
+
118
+
if (!$format->isExportFormatEnabled()) {
119
+
throw new PhutilArgumentUsageException(
120
+
pht(
121
+
'Export format ("%s") is not enabled.',
122
+
$format_key));
123
+
}
124
+
125
+
$is_overwrite = $args->getArg('overwrite');
126
+
$output_path = $args->getArg('output');
127
+
128
+
if (!strlen($output_path) && $is_overwrite) {
129
+
throw new PhutilArgumentUsageException(
130
+
pht(
131
+
'Flag "--overwrite" has no effect without "--output".'));
132
+
}
133
+
134
+
if (!$is_overwrite) {
135
+
if (Filesystem::pathExists($output_path)) {
136
+
throw new PhutilArgumentUsageException(
137
+
pht(
138
+
'Output path already exists. Use "--overwrite" to overwrite '.
139
+
'it.'));
140
+
}
141
+
}
142
+
143
+
$export_engine = id(new PhabricatorExportEngine())
144
+
->setViewer($viewer)
145
+
->setTitle(pht('Export'))
146
+
->setFilename(pht('export'))
147
+
->setSearchEngine($engine)
148
+
->setSavedQuery($saved_query)
149
+
->setExportFormat($format);
150
+
151
+
$file = $export_engine->exportFile();
152
+
153
+
$iterator = $file->getFileDataIterator();
154
+
155
+
if (strlen($output_path)) {
156
+
foreach ($iterator as $chunk) {
157
+
Filesystem::appendFile($output_path, $chunk);
158
+
}
159
+
} else {
160
+
foreach ($iterator as $chunk) {
161
+
echo $chunk;
162
+
}
163
+
}
164
+
165
+
return 0;
166
+
}
167
+
168
+
}