@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Properly create Elasticsearch index
Summary:
When the index does not exist and auto_create_index isn't
enabled, running ./bin/index results in a failure. That's
T5990
Instead create an index properly. This also allows us to do
nice things like do a proper mapping and analysis like for
substring matching like outlined by @fabe in T6552.
Test Plan:
Deleted and created index multiple times to verify
proper index creation and usage.
Reviewers: epriestley, #blessed_reviewers
Reviewed By: epriestley, #blessed_reviewers
Subscribers: Korvin, manybubbles, chasemp, fabe, epriestley
Differential Revision: https://secure.phabricator.com/D10955
authored by
Chad Horohoe
and committed by
epriestley
a366f85c
c953c0fe
+247
-8
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
2310
2310
'PhabricatorSearchHovercardController' => 'applications/search/controller/PhabricatorSearchHovercardController.php',
2311
2311
'PhabricatorSearchIndexer' => 'applications/search/index/PhabricatorSearchIndexer.php',
2312
2312
'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
2313
+
'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php',
2313
2314
'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
2314
2315
'PhabricatorSearchOrderController' => 'applications/search/controller/PhabricatorSearchOrderController.php',
2315
2316
'PhabricatorSearchRelationship' => 'applications/search/constants/PhabricatorSearchRelationship.php',
···
2349
2350
'PhabricatorSetupCheckBinaries' => 'applications/config/check/PhabricatorSetupCheckBinaries.php',
2350
2351
'PhabricatorSetupCheckDaemons' => 'applications/config/check/PhabricatorSetupCheckDaemons.php',
2351
2352
'PhabricatorSetupCheckDatabase' => 'applications/config/check/PhabricatorSetupCheckDatabase.php',
2353
+
'PhabricatorSetupCheckElastic' => 'applications/config/check/PhabricatorSetupCheckElastic.php',
2352
2354
'PhabricatorSetupCheckExtensions' => 'applications/config/check/PhabricatorSetupCheckExtensions.php',
2353
2355
'PhabricatorSetupCheckExtraConfig' => 'applications/config/check/PhabricatorSetupCheckExtraConfig.php',
2354
2356
'PhabricatorSetupCheckFileinfo' => 'applications/config/check/PhabricatorSetupCheckFileinfo.php',
···
5515
5517
'PhabricatorSearchEngineMySQL' => 'PhabricatorSearchEngine',
5516
5518
'PhabricatorSearchHovercardController' => 'PhabricatorSearchBaseController',
5517
5519
'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
5520
+
'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow',
5518
5521
'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
5519
5522
'PhabricatorSearchOrderController' => 'PhabricatorSearchBaseController',
5520
5523
'PhabricatorSearchResultView' => 'AphrontView',
···
5551
5554
'PhabricatorSetupCheckBinaries' => 'PhabricatorSetupCheck',
5552
5555
'PhabricatorSetupCheckDaemons' => 'PhabricatorSetupCheck',
5553
5556
'PhabricatorSetupCheckDatabase' => 'PhabricatorSetupCheck',
5557
+
'PhabricatorSetupCheckElastic' => 'PhabricatorSetupCheck',
5554
5558
'PhabricatorSetupCheckExtensions' => 'PhabricatorSetupCheck',
5555
5559
'PhabricatorSetupCheckExtraConfig' => 'PhabricatorSetupCheck',
5556
5560
'PhabricatorSetupCheckFileinfo' => 'PhabricatorSetupCheck',
+39
src/applications/config/check/PhabricatorSetupCheckElastic.php
+39
src/applications/config/check/PhabricatorSetupCheckElastic.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSetupCheckElastic extends PhabricatorSetupCheck {
4
+
5
+
protected function executeChecks() {
6
+
if (PhabricatorDefaultSearchEngineSelector::shouldUseElasticSearch()) {
7
+
$engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
8
+
if (!$engine->indexExists()) {
9
+
$summary = pht(
10
+
'You enabled Elasticsearch but the index does not exist.');
11
+
12
+
$message = pht(
13
+
'You likely enabled search.elastic.host without creating the '.
14
+
'index. Run `./bin/search init` to correct the index.');
15
+
16
+
$this
17
+
->newIssue('elastic.missing-index')
18
+
->setName(pht('Elasticsearch index Not Found'))
19
+
->setSummary($summary)
20
+
->setMessage($message)
21
+
->addRelatedPhabricatorConfig('search.elastic.host');
22
+
} else if (!$engine->indexIsSane()) {
23
+
$summary = pht(
24
+
'Elasticsearch index exists but needs correction.');
25
+
26
+
$message = pht(
27
+
'Either the Phabricator schema for Elasticsearch has changed '.
28
+
'or Elasticsearch created the index automatically. Run '.
29
+
'`./bin/search init` to correct the index.');
30
+
31
+
$this
32
+
->newIssue('elastic.broken-index')
33
+
->setName(pht('Elasticsearch index Incorrect'))
34
+
->setSummary($summary)
35
+
->setMessage($message);
36
+
}
37
+
}
38
+
}
39
+
}
+22
src/applications/search/engine/PhabricatorSearchEngine.php
+22
src/applications/search/engine/PhabricatorSearchEngine.php
···
36
36
*/
37
37
abstract public function executeSearch(PhabricatorSavedQuery $query);
38
38
39
+
/**
40
+
* Does the search index exist?
41
+
*
42
+
* @return bool
43
+
*/
44
+
abstract public function indexExists();
45
+
46
+
/**
47
+
* Is the index in a usable state?
48
+
*
49
+
* @return bool
50
+
*/
51
+
public function indexIsSane() {
52
+
return $this->indexExists();
53
+
}
54
+
55
+
/**
56
+
* Do any sort of setup for the search index
57
+
*
58
+
* @return void
59
+
*/
60
+
public function initIndex() {}
39
61
}
+129
-8
src/applications/search/engine/PhabricatorSearchEngineElastic.php
+129
-8
src/applications/search/engine/PhabricatorSearchEngineElastic.php
···
52
52
);
53
53
}
54
54
55
-
$this->executeRequest(
56
-
"/{$type}/{$phid}/",
57
-
$spec,
58
-
$is_write = true);
55
+
$this->executeRequest("/{$type}/{$phid}/", $spec, 'PUT');
59
56
}
60
57
61
58
public function reconstructDocument($phid) {
···
236
233
return $phids;
237
234
}
238
235
239
-
private function executeRequest($path, array $data, $is_write = false) {
236
+
public function indexExists() {
237
+
try {
238
+
return (bool)$this->executeRequest('/_status/', array());
239
+
} catch (HTTPFutureHTTPResponseStatus $e) {
240
+
if ($e->getStatusCode() == 404) {
241
+
return false;
242
+
}
243
+
throw $e;
244
+
}
245
+
}
246
+
247
+
private function getIndexConfiguration() {
248
+
$data = array();
249
+
$data['settings'] = array(
250
+
'index' => array(
251
+
'auto_expand_replicas' => '0-2',
252
+
'analysis' => array(
253
+
'filter' => array(
254
+
'trigrams_filter' => array(
255
+
'min_gram' => 3,
256
+
'type' => 'ngram',
257
+
'max_gram' => 3,
258
+
),
259
+
),
260
+
'analyzer' => array(
261
+
'custom_trigrams' => array(
262
+
'type' => 'custom',
263
+
'filter' => array(
264
+
'lowercase',
265
+
'kstem',
266
+
'trigrams_filter',
267
+
),
268
+
'tokenizer' => 'standard',
269
+
),
270
+
),
271
+
),
272
+
),
273
+
);
274
+
275
+
$types = array_keys(
276
+
PhabricatorSearchApplicationSearchEngine::getIndexableDocumentTypes());
277
+
foreach ($types as $type) {
278
+
$data['mappings'][$type]['properties']['field']['properties']['corpus'] =
279
+
array( 'type' => 'string', 'analyzer' => 'custom_trigrams' );
280
+
}
281
+
282
+
return $data;
283
+
}
284
+
285
+
public function indexIsSane() {
286
+
if (!$this->indexExists()) {
287
+
return false;
288
+
}
289
+
290
+
$cur_mapping = $this->executeRequest('/_mapping/', array());
291
+
$cur_settings = $this->executeRequest('/_settings/', array());
292
+
$actual = array_merge($cur_settings[$this->index],
293
+
$cur_mapping[$this->index]);
294
+
295
+
return $this->check($actual, $this->getIndexConfiguration());
296
+
}
297
+
298
+
/**
299
+
* Recursively check if two Elasticsearch configuration arrays are equal
300
+
*
301
+
* @param $actual
302
+
* @param $required array
303
+
* @return bool
304
+
*/
305
+
private function check($actual, $required) {
306
+
foreach ($required as $key => $value) {
307
+
if (!array_key_exists($key, $actual)) {
308
+
if ($key === '_all') {
309
+
// The _all field never comes back so we just have to assume it
310
+
// is set correctly.
311
+
continue;
312
+
}
313
+
return false;
314
+
}
315
+
if (is_array($value)) {
316
+
if (!is_array($actual[$key])) {
317
+
return false;
318
+
}
319
+
if (!$this->check($actual[$key], $value)) {
320
+
return false;
321
+
}
322
+
continue;
323
+
}
324
+
325
+
$actual[$key] = self::normalizeConfigValue($actual[$key]);
326
+
$value = self::normalizeConfigValue($value);
327
+
if ($actual[$key] != $value) {
328
+
return false;
329
+
}
330
+
}
331
+
return true;
332
+
}
333
+
334
+
/**
335
+
* Normalize a config value for comparison. Elasticsearch accepts all kinds
336
+
* of config values but it tends to throw back 'true' for true and 'false' for
337
+
* false so we normalize everything. Sometimes, oddly, it'll throw back false
338
+
* for false....
339
+
*
340
+
* @param mixed $value config value
341
+
* @return mixed value normalized
342
+
*/
343
+
private static function normalizeConfigValue($value) {
344
+
if ($value === true) {
345
+
return 'true';
346
+
} else if ($value === false) {
347
+
return 'false';
348
+
}
349
+
return $value;
350
+
}
351
+
352
+
public function initIndex() {
353
+
if ($this->indexExists()) {
354
+
$this->executeRequest('/', array(), 'DELETE');
355
+
}
356
+
$data = $this->getIndexConfiguration();
357
+
$this->executeRequest('/', $data, 'PUT');
358
+
}
359
+
360
+
private function executeRequest($path, array $data, $method = 'GET') {
240
361
$uri = new PhutilURI($this->uri);
241
362
$uri->setPath($this->index);
242
363
$uri->appendPath($path);
243
364
$data = json_encode($data);
244
365
245
366
$future = new HTTPSFuture($uri, $data);
246
-
if ($is_write) {
247
-
$future->setMethod('PUT');
367
+
if ($method != 'GET') {
368
+
$future->setMethod($method);
248
369
}
249
370
if ($this->getTimeout()) {
250
371
$future->setTimeout($this->getTimeout());
251
372
}
252
373
list($body) = $future->resolvex();
253
374
254
-
if ($is_write) {
375
+
if ($method != 'GET') {
255
376
return null;
256
377
}
257
378
+3
src/applications/search/engine/PhabricatorSearchEngineMySQL.php
+3
src/applications/search/engine/PhabricatorSearchEngineMySQL.php
+50
src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
+50
src/applications/search/management/PhabricatorSearchManagementInitWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhabricatorSearchManagementInitWorkflow
4
+
extends PhabricatorSearchManagementWorkflow {
5
+
6
+
protected function didConstruct() {
7
+
$this
8
+
->setName('init')
9
+
->setSynopsis('Initialize or repair an index.')
10
+
->setExamples('**init**');
11
+
}
12
+
13
+
public function execute(PhutilArgumentParser $args) {
14
+
$console = PhutilConsole::getConsole();
15
+
16
+
$engine = PhabricatorSearchEngineSelector::newSelector()->newEngine();
17
+
18
+
$work_done = false;
19
+
if (!$engine->indexExists()) {
20
+
$console->writeOut(
21
+
'%s',
22
+
pht('Index does not exist, creating...'));
23
+
$engine->initIndex();
24
+
$console->writeOut(
25
+
"%s\n",
26
+
pht('done.'));
27
+
$work_done = true;
28
+
} else if (!$engine->indexIsSane()) {
29
+
$console->writeOut(
30
+
'%s',
31
+
pht('Index exists but is incorrect, fixing...'));
32
+
$engine->initIndex();
33
+
$console->writeOut(
34
+
"%s\n",
35
+
pht('done.'));
36
+
$work_done = true;
37
+
}
38
+
39
+
if ($work_done) {
40
+
$console->writeOut(
41
+
"%s\n",
42
+
pht('Index maintenance complete. Run `./bin/search index` to '.
43
+
'reindex documents'));
44
+
} else {
45
+
$console->writeOut(
46
+
"%s\n",
47
+
pht('Nothing to do.'));
48
+
}
49
+
}
50
+
}