@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
1<?php
2
3final class PhabricatorSearchManagementIndexWorkflow
4 extends PhabricatorSearchManagementWorkflow {
5
6 protected function didConstruct() {
7 $this
8 ->setName('index')
9 ->setSynopsis(pht('Build or rebuild search indexes.'))
10 ->setExamples(
11 implode(
12 "\n",
13 array(
14 '**index** D123',
15 '**index** --all',
16 '**index** [--type __task__] [--version __version__] ...',
17 )))
18 ->setArguments(
19 array(
20 array(
21 'name' => 'all',
22 'help' => pht('Reindex all documents.'),
23 ),
24 array(
25 'name' => 'type',
26 'param' => 'type',
27 'repeat' => true,
28 'help' => pht(
29 'Object types to reindex, like "task", "commit" or "revision".'),
30 ),
31 array(
32 'name' => 'background',
33 'help' => pht(
34 'Instead of indexing in this process, queue tasks for '.
35 'the daemons. This can improve performance, but makes '.
36 'it more difficult to debug search indexing.'),
37 ),
38 array(
39 'name' => 'force',
40 'short' => 'f',
41 'help' => pht(
42 'Force a complete rebuild of the entire index instead of an '.
43 'incremental update.'),
44 ),
45 array(
46 'name' => 'version',
47 'param' => 'version',
48 'repeat' => true,
49 'help' => pht(
50 'Reindex objects previously indexed with a particular '.
51 'version of the indexer.'),
52 ),
53 array(
54 'name' => 'min-index-date',
55 'param' => 'date',
56 'help' => pht(
57 'Reindex objects previously indexed on or after a '.
58 'given date.'),
59 ),
60 array(
61 'name' => 'max-index-date',
62 'param' => 'date',
63 'help' => pht(
64 'Reindex objects previously indexed on or before a '.
65 'given date.'),
66 ),
67 array(
68 'name' => 'objects',
69 'wildcard' => true,
70 ),
71 ));
72 }
73
74 public function execute(PhutilArgumentParser $args) {
75 $this->validateClusterSearchConfig();
76
77 $is_all = $args->getArg('all');
78 $is_force = $args->getArg('force');
79
80 $object_types = $args->getArg('type');
81 $index_versions = $args->getArg('version');
82
83 $min_epoch = $args->getArg('min-index-date');
84 if ($min_epoch !== null) {
85 $min_epoch = $this->parseTimeArgument($min_epoch);
86 }
87
88 $max_epoch = $args->getArg('max-index-date');
89 if ($max_epoch !== null) {
90 $max_epoch = $this->parseTimeArgument($max_epoch);
91 }
92
93 $object_names = $args->getArg('objects');
94
95 $any_constraints =
96 ($object_names) ||
97 ($object_types) ||
98 ($index_versions) ||
99 ($min_epoch) ||
100 ($max_epoch);
101
102 if ($is_all && $any_constraints) {
103 throw new PhutilArgumentUsageException(
104 pht(
105 'You can not use query constraint flags (like "--version", '.
106 '"--type", or a list of specific objects) with "--all".'));
107 }
108
109 if (!$is_all && !$any_constraints) {
110 throw new PhutilArgumentUsageException(
111 pht(
112 'Provide a list of objects to index (like "D123"), or a set of '.
113 'query constraint flags (like "--type"), or "--all" to index '.
114 'all objects.'));
115 }
116
117
118 if ($args->getArg('background')) {
119 $is_background = true;
120 } else {
121 PhabricatorWorker::setRunAllTasksInProcess(true);
122 $is_background = false;
123 }
124
125 if (!$is_background) {
126 $this->logInfo(
127 pht('NOTE'),
128 pht(
129 'Run this workflow with "--background" to queue tasks for the '.
130 'daemon workers.'));
131 }
132
133 $this->logInfo(
134 pht('SELECT'),
135 pht('Selecting objects to index...'));
136
137 $object_phids = null;
138 if ($object_names) {
139 $object_phids = $this->loadPHIDsByNames($object_names);
140 $object_phids = array_fuse($object_phids);
141 }
142
143 $type_phids = null;
144 if ($is_all || $object_types) {
145 $object_map = $this->getIndexableObjectsByTypes($object_types);
146 $type_phids = array();
147 foreach ($object_map as $object) {
148 $iterator = new LiskMigrationIterator($object);
149 foreach ($iterator as $o) {
150 $type_phids[] = $o->getPHID();
151 }
152 }
153 $type_phids = array_fuse($type_phids);
154 }
155
156 $index_phids = null;
157 if ($index_versions || $min_epoch || $max_epoch) {
158 $index_phids = $this->loadPHIDsByIndexConstraints(
159 $index_versions,
160 $min_epoch,
161 $max_epoch);
162 $index_phids = array_fuse($index_phids);
163 }
164
165 $working_set = null;
166 $filter_sets = array(
167 $object_phids,
168 $type_phids,
169 $index_phids,
170 );
171
172 foreach ($filter_sets as $filter_set) {
173 if ($filter_set === null) {
174 continue;
175 }
176
177 if ($working_set === null) {
178 $working_set = $filter_set;
179 continue;
180 }
181
182 $working_set = array_intersect_key($working_set, $filter_set);
183 }
184
185 $phids = array_keys($working_set);
186
187 if (!$phids) {
188 $this->logWarn(
189 pht('NO OBJECTS'),
190 pht('No objects selected to index.'));
191 return 0;
192 }
193
194 $this->logInfo(
195 pht('INDEXING'),
196 pht(
197 'Indexing %s object(s).',
198 phutil_count($phids)));
199
200 $bar = id(new PhutilConsoleProgressBar())
201 ->setTotal(count($phids));
202
203 $parameters = array(
204 'force' => $is_force,
205 );
206
207 $any_success = false;
208
209 // If we aren't using "--background" or "--force", track how many objects
210 // we're skipping so we can print this information for the user and give
211 // them a hint that they might want to use "--force".
212 $track_skips = (!$is_background && !$is_force);
213
214 // Activate "strict" error reporting if we're running in the foreground
215 // so we'll report a wider range of conditions as errors.
216 $is_strict = !$is_background;
217
218 $count_updated = 0;
219 $count_skipped = 0;
220
221 foreach ($phids as $phid) {
222 try {
223 if ($track_skips) {
224 $old_versions = $this->loadIndexVersions($phid);
225 }
226
227 PhabricatorSearchWorker::queueDocumentForIndexing(
228 $phid,
229 $parameters,
230 $is_strict);
231
232 if ($track_skips) {
233 $new_versions = $this->loadIndexVersions($phid);
234
235 if (!$old_versions && !$new_versions) {
236 // If the document doesn't use an index version, both the lists
237 // of versions will be empty. We still rebuild the index in this
238 // case.
239 $count_updated++;
240 } else if ($old_versions !== $new_versions) {
241 $count_updated++;
242 } else {
243 $count_skipped++;
244 }
245 }
246
247 $any_success = true;
248 } catch (Exception $ex) {
249 phlog($ex);
250 }
251
252 $bar->update(1);
253 }
254
255 $bar->done();
256
257 if (!$any_success) {
258 throw new Exception(
259 pht('Failed to rebuild search index for any documents.'));
260 }
261
262 if ($track_skips) {
263 if ($count_updated) {
264 $this->logOkay(
265 pht('DONE'),
266 pht(
267 'Updated search indexes for %s document(s).',
268 new PhutilNumber($count_updated)));
269 }
270
271 if ($count_skipped) {
272 $this->logWarn(
273 pht('SKIP'),
274 pht(
275 'Skipped %s document(s) which have not updated since they were '.
276 'last indexed.',
277 new PhutilNumber($count_skipped)));
278 $this->logInfo(
279 pht('NOTE'),
280 pht(
281 'Use "--force" to force the index to update these documents.'));
282 }
283 } else if ($is_background) {
284 $this->logOkay(
285 pht('DONE'),
286 pht(
287 'Queued %s document(s) for background indexing.',
288 new PhutilNumber(count($phids))));
289 } else {
290 $this->logOkay(
291 pht('DONE'),
292 pht(
293 'Forced search index updates for %s document(s).',
294 new PhutilNumber(count($phids))));
295 }
296 }
297
298 private function loadPHIDsByNames(array $names) {
299 $query = id(new PhabricatorObjectQuery())
300 ->setViewer($this->getViewer())
301 ->withNames($names);
302 $query->execute();
303 $objects = $query->getNamedResults();
304
305 foreach ($names as $name) {
306 if (empty($objects[$name])) {
307 throw new PhutilArgumentUsageException(
308 pht(
309 "'%s' is not the name of a known object.",
310 $name));
311 }
312 }
313
314 return mpull($objects, 'getPHID');
315 }
316
317 private function getIndexableObjectsByTypes(array $types) {
318 $objects = id(new PhutilClassMapQuery())
319 ->setAncestorClass(PhabricatorIndexableInterface::class)
320 ->execute();
321
322 $type_map = array();
323 $normal_map = array();
324 foreach ($types as $type) {
325 $normalized_type = phutil_utf8_strtolower($type);
326 $type_map[$type] = $normalized_type;
327
328 if (isset($normal_map[$normalized_type])) {
329 $old_type = $normal_map[$normalized_type];
330 throw new PhutilArgumentUsageException(
331 pht(
332 'Type specification "%s" duplicates type specification "%s". '.
333 'Specify each type only once.',
334 $type,
335 $old_type));
336 }
337
338 $normal_map[$normalized_type] = $type;
339 }
340
341 $object_matches = array();
342
343 $matches_map = array();
344 $exact_map = array();
345 foreach ($objects as $object) {
346 $object_class = get_class($object);
347
348 if (!$types) {
349 $object_matches[$object_class] = $object;
350 continue;
351 }
352
353 $normalized_class = phutil_utf8_strtolower($object_class);
354
355 // If a specified type is exactly the name of this class, match it.
356 if (isset($normal_map[$normalized_class])) {
357 $object_matches[$object_class] = $object;
358 $matching_type = $normal_map[$normalized_class];
359 $matches_map[$matching_type] = array($object_class);
360 $exact_map[$matching_type] = true;
361 continue;
362 }
363
364 foreach ($type_map as $type => $normalized_type) {
365 // If we already have an exact match for this type, don't match it
366 // as a substring. An indexable "MothObject" should be selectable
367 // exactly without also selecting "MammothObject".
368 if (isset($exact_map[$type])) {
369 continue;
370 }
371
372 // If the selector isn't a substring of the class name, continue.
373 if (strpos($normalized_class, $normalized_type) === false) {
374 continue;
375 }
376
377 $matches_map[$type][] = $object_class;
378 $object_matches[$object_class] = $object;
379 }
380 }
381
382 $all_types = array();
383 foreach ($objects as $object) {
384 $all_types[] = get_class($object);
385 }
386 sort($all_types);
387 $type_list = implode(', ', $all_types);
388
389 foreach ($type_map as $type => $normalized_type) {
390 $matches = idx($matches_map, $type);
391 if (!$matches) {
392 throw new PhutilArgumentUsageException(
393 pht(
394 'Type "%s" matches no indexable objects. '.
395 'Supported types are: %s.',
396 $type,
397 $type_list));
398 }
399
400 if (count($matches) > 1) {
401 throw new PhutilArgumentUsageException(
402 pht(
403 'Type "%s" matches multiple indexable objects. Use a more '.
404 'specific string. Matching objects are: %s.',
405 $type,
406 implode(', ', $matches)));
407 }
408 }
409
410 return $object_matches;
411 }
412
413 private function loadIndexVersions($phid) {
414 $table = new PhabricatorSearchIndexVersion();
415 $conn = $table->establishConnection('r');
416
417 return queryfx_all(
418 $conn,
419 'SELECT extensionKey, version FROM %T WHERE objectPHID = %s
420 ORDER BY extensionKey, version',
421 $table->getTableName(),
422 $phid);
423 }
424
425 private function loadPHIDsByIndexConstraints(
426 array $index_versions,
427 $min_date,
428 $max_date) {
429
430 $table = new PhabricatorSearchIndexVersion();
431 $conn = $table->establishConnection('r');
432
433 $where = array();
434 if ($index_versions) {
435 $where[] = qsprintf(
436 $conn,
437 'indexVersion IN (%Ls)',
438 $index_versions);
439 }
440
441 if ($min_date !== null) {
442 $where[] = qsprintf(
443 $conn,
444 'indexEpoch >= %d',
445 $min_date);
446 }
447
448 if ($max_date !== null) {
449 $where[] = qsprintf(
450 $conn,
451 'indexEpoch <= %d',
452 $max_date);
453 }
454
455 $rows = queryfx_all(
456 $conn,
457 'SELECT DISTINCT objectPHID FROM %R WHERE %LA',
458 $table,
459 $where);
460
461 return ipull($rows, 'objectPHID');
462 }
463
464}