@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Allow "bin/bulk export" to merge multiple queries and accept more flexible flags
Summary:
Ref T13210. Minor usability improvements to "bin/bulk export":
- Allow `--class task` to work (previously, only `--class ManiphestTaskSearchEngine` worked).
- If you run `--query jXIlzQyOYHPU`, don't require `--class`, since the query identifies the class on its own.
- Allow users to call `--query A --query B --query C` and get a union of all results.
Test Plan:
- Ran `--class task`, `--query A --query B`, `--query X` (with no `--class`), got good results.
- Ran various flavors of bad combinations (queries from different engines, invalid engines, query and class differing, ambiguous/invalid `--class` name) and got sensible errors.
Reviewers: amckinley
Reviewed By: amckinley
Maniphest Tasks: T13210
Differential Revision: https://secure.phabricator.com/D19738
+188
-52
+187
-51
src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php
+187
-51
src/applications/transactions/bulk/management/PhabricatorBulkManagementExportWorkflow.php
···
26
26
'name' => 'query',
27
27
'param' => 'key',
28
28
'help' => pht(
29
-
'Export the data selected by this query.'),
29
+
'Export the data selected by one or more queries.'),
30
+
'repeat' => true,
30
31
),
31
32
array(
32
33
'name' => 'output',
···
47
48
public function execute(PhutilArgumentParser $args) {
48
49
$viewer = $this->getViewer();
49
50
50
-
$class = $args->getArg('class');
51
-
52
-
if (!strlen($class)) {
53
-
throw new PhutilArgumentUsageException(
54
-
pht(
55
-
'Specify a search engine class to export data from with '.
56
-
'"--class".'));
57
-
}
58
-
59
-
if (!is_subclass_of($class, 'PhabricatorApplicationSearchEngine')) {
60
-
throw new PhutilArgumentUsageException(
61
-
pht(
62
-
'SearchEngine class ("%s") is unknown.',
63
-
$class));
64
-
}
65
-
66
-
$engine = newv($class, array())
67
-
->setViewer($viewer);
68
-
69
-
if (!$engine->canExport()) {
70
-
throw new PhutilArgumentUsageException(
71
-
pht(
72
-
'SearchEngine class ("%s") does not support data export.',
73
-
$class));
74
-
}
75
-
76
-
$query_key = $args->getArg('query');
77
-
if (!strlen($query_key)) {
78
-
throw new PhutilArgumentUsageException(
79
-
pht(
80
-
'Specify a query to export with "--query".'));
81
-
}
82
-
83
-
if ($engine->isBuiltinQuery($query_key)) {
84
-
$saved_query = $engine->buildSavedQueryFromBuiltin($query_key);
85
-
} else if ($query_key) {
86
-
$saved_query = id(new PhabricatorSavedQueryQuery())
87
-
->setViewer($viewer)
88
-
->withQueryKeys(array($query_key))
89
-
->executeOne();
90
-
} else {
91
-
$saved_query = null;
92
-
}
93
-
94
-
if (!$saved_query) {
95
-
throw new PhutilArgumentUsageException(
96
-
pht(
97
-
'Failed to load saved query ("%s").',
98
-
$query_key));
99
-
}
51
+
list($engine, $queries) = $this->newQueries($args);
100
52
101
53
$format_key = $args->getArg('format');
102
54
if (!strlen($format_key)) {
···
140
92
}
141
93
}
142
94
95
+
// If we have more than one query, execute the queries to figure out which
96
+
// results they hit, then build a synthetic query for all those results
97
+
// using the IDs.
98
+
if (count($queries) > 1) {
99
+
$saved_query = $this->newUnionQuery($engine, $queries);
100
+
} else {
101
+
$saved_query = head($queries);
102
+
}
103
+
143
104
$export_engine = id(new PhabricatorExportEngine())
144
105
->setViewer($viewer)
145
106
->setTitle(pht('Export'))
···
163
124
}
164
125
165
126
return 0;
127
+
}
128
+
129
+
private function newQueries(PhutilArgumentParser $args) {
130
+
$viewer = $this->getViewer();
131
+
132
+
$query_keys = $args->getArg('query');
133
+
if (!$query_keys) {
134
+
throw new PhutilArgumentUsageException(
135
+
pht(
136
+
'Specify one or more queries to export with "--query".'));
137
+
}
138
+
139
+
$engine_classes = id(new PhutilClassMapQuery())
140
+
->setAncestorClass('PhabricatorApplicationSearchEngine')
141
+
->execute();
142
+
143
+
$class = $args->getArg('class');
144
+
if (strlen($class)) {
145
+
146
+
$class_list = array();
147
+
foreach ($engine_classes as $class_name => $engine_object) {
148
+
$can_export = id(clone $engine_object)
149
+
->setViewer($viewer)
150
+
->canExport();
151
+
if ($can_export) {
152
+
$class_list[] = $class_name;
153
+
}
154
+
}
155
+
156
+
sort($class_list);
157
+
$class_list = implode(', ', $class_list);
158
+
159
+
$matches = array();
160
+
foreach ($engine_classes as $class_name => $engine_object) {
161
+
if (stripos($class_name, $class) !== false) {
162
+
if (strtolower($class_name) == strtolower($class)) {
163
+
$matches = array($class_name);
164
+
break;
165
+
} else {
166
+
$matches[] = $class_name;
167
+
}
168
+
}
169
+
}
170
+
171
+
if (!$matches) {
172
+
throw new PhutilArgumentUsageException(
173
+
pht(
174
+
'No search engines match "%s". Available engines which support '.
175
+
'data export are: %s.',
176
+
$class,
177
+
$class_list));
178
+
} else if (count($matches) > 1) {
179
+
throw new PhutilArgumentUsageException(
180
+
pht(
181
+
'Multiple search engines match "%s": %s.',
182
+
$class,
183
+
implode(', ', $matches)));
184
+
} else {
185
+
$class = head($matches);
186
+
}
187
+
188
+
$engine = newv($class, array())
189
+
->setViewer($viewer);
190
+
} else {
191
+
$engine = null;
192
+
}
193
+
194
+
$queries = array();
195
+
foreach ($query_keys as $query_key) {
196
+
if ($engine) {
197
+
if ($engine->isBuiltinQuery($query_key)) {
198
+
$queries[$query_key] = $engine->buildSavedQueryFromBuiltin(
199
+
$query_key);
200
+
continue;
201
+
}
202
+
}
203
+
204
+
$saved_query = id(new PhabricatorSavedQueryQuery())
205
+
->setViewer($viewer)
206
+
->withQueryKeys(array($query_key))
207
+
->executeOne();
208
+
if (!$saved_query) {
209
+
if (!$engine) {
210
+
throw new PhutilArgumentUsageException(
211
+
pht(
212
+
'Query "%s" is unknown. To run a builtin query like "all" or '.
213
+
'"active", also specify the search engine with "--class".',
214
+
$query_key));
215
+
} else {
216
+
throw new PhutilArgumentUsageException(
217
+
pht(
218
+
'Query "%s" is not a recognized query for class "%s".',
219
+
$query_key,
220
+
get_class($engine)));
221
+
}
222
+
}
223
+
224
+
$queries[$query_key] = $saved_query;
225
+
}
226
+
227
+
// If we don't have an engine from "--class", fill it in by looking at the
228
+
// class of the first query.
229
+
if (!$engine) {
230
+
foreach ($queries as $query) {
231
+
$engine = newv($query->getEngineClassName(), array())
232
+
->setViewer($viewer);
233
+
break;
234
+
}
235
+
}
236
+
237
+
$engine_class = get_class($engine);
238
+
239
+
foreach ($queries as $query) {
240
+
$query_class = $query->getEngineClassName();
241
+
if ($query_class !== $engine_class) {
242
+
throw new PhutilArgumentUsageException(
243
+
pht(
244
+
'Specified queries use different engines: query "%s" uses '.
245
+
'engine "%s", not "%s". All queries must run on the same '.
246
+
'engine.',
247
+
$query->getQueryKey(),
248
+
$query_class,
249
+
$engine_class));
250
+
}
251
+
}
252
+
253
+
if (!$engine->canExport()) {
254
+
throw new PhutilArgumentUsageException(
255
+
pht(
256
+
'SearchEngine class ("%s") does not support data export.',
257
+
$engine_class));
258
+
}
259
+
260
+
return array($engine, $queries);
261
+
}
262
+
263
+
private function newUnionQuery(
264
+
PhabricatorApplicationSearchEngine $engine,
265
+
array $queries) {
266
+
267
+
assert_instances_of($queries, 'PhabricatorSavedQuery');
268
+
269
+
$engine = clone $engine;
270
+
271
+
$ids = array();
272
+
foreach ($queries as $saved_query) {
273
+
$page_size = 1000;
274
+
$page_cursor = null;
275
+
do {
276
+
$query = $engine->buildQueryFromSavedQuery($saved_query);
277
+
$pager = $engine->newPagerForSavedQuery($saved_query);
278
+
$pager->setPageSize($page_size);
279
+
280
+
if ($page_cursor !== null) {
281
+
$pager->setAfterID($page_cursor);
282
+
}
283
+
284
+
$objects = $engine->executeQuery($query, $pager);
285
+
$page_cursor = $pager->getNextPageID();
286
+
287
+
foreach ($objects as $object) {
288
+
$ids[] = $object->getID();
289
+
}
290
+
} while ($pager->getHasMoreResults());
291
+
}
292
+
293
+
// When we're merging multiple different queries, override any query order
294
+
// and just put the combined result list in ID order. At time of writing,
295
+
// we can't merge the result sets together while retaining the overall sort
296
+
// order even if they all used the same order, and it's meaningless to try
297
+
// to retain orders if the queries had different orders in the first place.
298
+
rsort($ids);
299
+
300
+
return id($engine->newSavedQuery())
301
+
->setParameter('ids', $ids);
166
302
}
167
303
168
304
}
+1
-1
src/infrastructure/export/engine/PhabricatorExportEngine.php
+1
-1
src/infrastructure/export/engine/PhabricatorExportEngine.php
···
125
125
$field_list = mpull($field_list, null, 'getKey');
126
126
$format->addHeaders($field_list);
127
127
128
-
// Iterate over the query results in large page so we don't have to hold
128
+
// Iterate over the query results in large pages so we don't have to hold
129
129
// too much stuff in memory.
130
130
$page_size = 1000;
131
131
$page_cursor = null;