@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Improve elasticsearch
Summary: I thought that this will be fun but the elasticsearch API is horrible and the documentation is poor.
Test Plan:
Search for:
- string
- author
- author, owner
- string, author
- open
- string, open, author
- string, exclude
- several authors, several owners
- nothing
- probably all other combinations
Normally, such an exhaustive test plan wouldn't be required but each combination requires a completely different query.
Reviewers: epriestley, jungejason
Reviewed By: epriestley
CC: aran, Koolvin, btrahan
Differential Revision: https://secure.phabricator.com/D2298
+123
-65
+1
-8
src/applications/search/controller/search/PhabricatorSearchController.php
+1
-8
src/applications/search/controller/search/PhabricatorSearchController.php
···
113
113
}
114
114
}
115
115
116
-
$more = PhabricatorEnv::getEnvConfig('search.more-document-types', array());
117
-
118
116
$options = array(
119
117
'' => 'All Documents',
120
-
PhabricatorPHIDConstants::PHID_TYPE_DREV => 'Differential Revisions',
121
-
PhabricatorPHIDConstants::PHID_TYPE_CMIT => 'Repository Commits',
122
-
PhabricatorPHIDConstants::PHID_TYPE_TASK => 'Maniphest Tasks',
123
-
PhabricatorPHIDConstants::PHID_TYPE_WIKI => 'Phriction Documents',
124
-
PhabricatorPHIDConstants::PHID_TYPE_USER => 'Phabricator Users',
125
-
) + $more;
118
+
) + PhabricatorSearchAbstractDocument::getSupportedTypes();
126
119
127
120
$status_options = array(
128
121
0 => 'Open and Closed Documents',
+1
-1
src/applications/search/controller/search/__init__.php
+1
-1
src/applications/search/controller/search/__init__.php
···
18
18
phutil_require_module('phabricator', 'applications/search/constants/scope');
19
19
phutil_require_module('phabricator', 'applications/search/controller/base');
20
20
phutil_require_module('phabricator', 'applications/search/engine/jumpnav');
21
+
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
21
22
phutil_require_module('phabricator', 'applications/search/selector/base');
22
23
phutil_require_module('phabricator', 'applications/search/storage/query');
23
24
phutil_require_module('phabricator', 'applications/search/view/searchresult');
24
25
phutil_require_module('phabricator', 'infrastructure/celerity/api');
25
-
phutil_require_module('phabricator', 'infrastructure/env');
26
26
phutil_require_module('phabricator', 'view/control/pager');
27
27
phutil_require_module('phabricator', 'view/form/base');
28
28
phutil_require_module('phabricator', 'view/form/control/select');
+96
-53
src/applications/search/engine/elastic/PhabricatorSearchEngineElastic.php
+96
-53
src/applications/search/engine/elastic/PhabricatorSearchEngineElastic.php
···
1
1
<?php
2
2
3
3
/*
4
-
* Copyright 2011 Facebook, Inc.
4
+
* Copyright 2012 Facebook, Inc.
5
5
*
6
6
* Licensed under the Apache License, Version 2.0 (the "License");
7
7
* you may not use this file except in compliance with the License.
···
23
23
24
24
$type = $doc->getDocumentType();
25
25
$phid = $doc->getPHID();
26
+
$handle = PhabricatorObjectHandleData::loadOneHandle($phid);
26
27
27
28
$spec = array(
28
-
'phid' => $phid,
29
-
'type' => $type,
30
29
'title' => $doc->getDocumentTitle(),
31
-
'dateCreated' => date('c', $doc->getDocumentCreated()),
32
-
'dateModified' => date('c', $doc->getDocumentModified()),
30
+
'url' => PhabricatorEnv::getProductionURI($handle->getURI()),
31
+
'dateCreated' => $doc->getDocumentCreated(),
32
+
'_timestamp' => $doc->getDocumentModified(),
33
33
'field' => array(),
34
34
'relationship' => array(),
35
35
);
36
36
37
37
foreach ($doc->getFieldData() as $field) {
38
-
list($ftype, $corpus, $aux_phid) = $field;
39
-
$spec['field'][$ftype][] = array(
40
-
'corpus' => $corpus,
41
-
'aux' => $aux_phid,
42
-
);
38
+
$spec['field'][] = array_combine(array('type', 'corpus', 'aux'), $field);
43
39
}
44
40
45
41
foreach ($doc->getRelationshipData() as $relationship) {
···
47
43
$spec['relationship'][$rtype][] = array(
48
44
'phid' => $to_phid,
49
45
'phidType' => $to_type,
50
-
'when' => date('c', $time),
46
+
'when' => $time,
51
47
);
52
48
}
53
49
···
58
54
}
59
55
60
56
public function reconstructDocument($phid) {
57
+
$type = phid_get_type($phid);
61
58
62
-
$response = $this->executeRequest(
63
-
'/phabricator/_search',
64
-
array(
65
-
'query' => array(
66
-
'ids' => array(
67
-
'values' => array(
68
-
$phid,
69
-
),
70
-
),
71
-
),
72
-
),
73
-
$is_write = false);
59
+
$response = $this->executeRequest("/phabricator/{$type}/{$phid}", array());
74
60
75
-
$hit = $response['hits']['hits'][0]['_source'];
76
-
if (!$hit) {
61
+
if (empty($response['exists'])) {
77
62
return null;
78
63
}
79
64
65
+
$hit = $response['_source'];
66
+
80
67
$doc = new PhabricatorSearchAbstractDocument();
81
-
$doc->setPHID($hit['phid']);
82
-
$doc->setDocumentType($hit['type']);
68
+
$doc->setPHID($phid);
69
+
$doc->setDocumentType($response['_type']);
83
70
$doc->setDocumentTitle($hit['title']);
84
-
$doc->setDocumentCreated(strtotime($hit['dateCreated']));
85
-
$doc->setDocumentModified(strtotime($hit['dateModified']));
71
+
$doc->setDocumentCreated($hit['dateCreated']);
72
+
$doc->setDocumentModified($hit['_timestamp']);
86
73
87
-
foreach ($hit['field'] as $ftype => $fdefs) {
88
-
foreach ($fdefs as $fdef) {
89
-
$doc->addField(
90
-
$ftype,
91
-
$fdef['corpus'],
92
-
$fdef['aux']);
93
-
}
74
+
foreach ($hit['field'] as $fdef) {
75
+
$doc->addField($fdef['type'], $fdef['corpus'], $fdef['aux']);
94
76
}
95
77
96
78
foreach ($hit['relationship'] as $rtype => $rships) {
···
99
81
$rtype,
100
82
$rship['phid'],
101
83
$rship['phidType'],
102
-
strtotime($rship['when']));
84
+
$rship['when']);
103
85
}
104
86
}
105
87
···
108
90
109
91
public function executeSearch(PhabricatorSearchQuery $query) {
110
92
111
-
$spec = array(
112
-
'text' => array(
113
-
'_all' => $query->getQuery(),
114
-
),
115
-
);
93
+
$spec = array();
94
+
$filter = array();
95
+
96
+
if ($query->getQuery()) {
97
+
$spec[] = array(
98
+
'field' => array(
99
+
'field.corpus' => $query->getQuery(),
100
+
),
101
+
);
102
+
}
103
+
104
+
$exclude = $query->getParameter('exclude');
105
+
if ($exclude) {
106
+
$filter[] = array(
107
+
'not' => array(
108
+
'ids' => array(
109
+
'values' => array($exclude),
110
+
),
111
+
),
112
+
);
113
+
}
116
114
117
115
$type = $query->getParameter('type');
118
116
if ($type) {
119
117
$uri = "/phabricator/{$type}/_search";
120
118
} else {
121
-
$uri = "/phabricator/_search";
119
+
// Don't use '/phabricator/_search' for the case that there is something
120
+
// else in the index (for example if 'phabricator' is only an alias to
121
+
// some bigger index).
122
+
$types = PhabricatorSearchAbstractDocument::getSupportedTypes();
123
+
$uri = '/phabricator/' . implode(',', array_keys($types)) . '/_search';
122
124
}
123
125
124
-
$response = $this->executeRequest(
125
-
$uri,
126
-
array(
127
-
'query' => $spec,
128
-
),
129
-
$is_write = false);
126
+
$rel_mapping = array(
127
+
'author' => PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR,
128
+
'open' => PhabricatorSearchRelationship::RELATIONSHIP_OPEN,
129
+
'owner' => PhabricatorSearchRelationship::RELATIONSHIP_OWNER,
130
+
'project' => PhabricatorSearchRelationship::RELATIONSHIP_PROJECT,
131
+
'repository' => PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY,
132
+
);
133
+
foreach ($rel_mapping as $name => $field) {
134
+
$param = $query->getParameter($name);
135
+
if (is_array($param)) {
136
+
$should = array();
137
+
foreach ($param as $val) {
138
+
$should[] = array(
139
+
'text' => array(
140
+
"relationship.{$field}.phid" => array(
141
+
'query' => $val,
142
+
'type' => 'phrase',
143
+
),
144
+
),
145
+
);
146
+
}
147
+
$spec[] = array('bool' => array('should' => $should));
148
+
} else if ($param) {
149
+
$filter[] = array(
150
+
'exists' => array(
151
+
'field' => "relationship.{$field}.phid",
152
+
),
153
+
);
154
+
}
155
+
}
130
156
131
-
$phids = array();
132
-
foreach ($response['hits']['hits'] as $hit) {
133
-
$phids[] = $hit['_id'];
157
+
if ($spec) {
158
+
$spec = array('query' => array('bool' => array('must' => $spec)));
159
+
}
160
+
161
+
if ($filter) {
162
+
$filter = array('filter' => array('and' => $filter));
163
+
if ($spec) {
164
+
$spec = array(
165
+
'query' => array(
166
+
'filtered' => $spec + $filter,
167
+
),
168
+
);
169
+
} else {
170
+
$spec = $filter;
171
+
}
134
172
}
135
173
174
+
$spec['from'] = (int)$query->getParameter('offset', 0);
175
+
$spec['size'] = (int)$query->getParameter('limit', 0);
176
+
$response = $this->executeRequest($uri, $spec);
177
+
178
+
$phids = ipull($response['hits']['hits'], '_id');
136
179
return $phids;
137
180
}
138
181
139
-
private function executeRequest($path, array $data, $is_write) {
182
+
private function executeRequest($path, array $data, $is_write = false) {
140
183
$uri = PhabricatorEnv::getEnvConfig('search.elastic.host');
141
184
$uri = new PhutilURI($uri);
142
185
$data = json_encode($data);
+4
src/applications/search/engine/elastic/__init__.php
+4
src/applications/search/engine/elastic/__init__.php
···
6
6
7
7
8
8
9
+
phutil_require_module('phabricator', 'applications/phid/handle/data');
10
+
phutil_require_module('phabricator', 'applications/phid/utils');
11
+
phutil_require_module('phabricator', 'applications/search/constants/relationship');
9
12
phutil_require_module('phabricator', 'applications/search/engine/base');
10
13
phutil_require_module('phabricator', 'applications/search/index/abstractdocument');
11
14
phutil_require_module('phabricator', 'infrastructure/env');
···
13
16
phutil_require_module('phutil', 'future/http/http');
14
17
phutil_require_module('phutil', 'future/http/https');
15
18
phutil_require_module('phutil', 'parser/uri');
19
+
phutil_require_module('phutil', 'utils');
16
20
17
21
18
22
phutil_require_source('PhabricatorSearchEngineElastic.php');
+6
-1
src/applications/search/engine/mysql/PhabricatorSearchEngineMySQL.php
+6
-1
src/applications/search/engine/mysql/PhabricatorSearchEngineMySQL.php
···
305
305
return ipull($hits, 'phid');
306
306
}
307
307
308
-
protected function joinRelationship($conn, $query, $field, $type) {
308
+
protected function joinRelationship(
309
+
AphrontDatabaseConnection $conn,
310
+
PhabricatorSearchQuery $query,
311
+
$field,
312
+
$type) {
313
+
309
314
$phids = $query->getParameter($field, array());
310
315
if (!$phids) {
311
316
return null;
+12
-1
src/applications/search/index/abstractdocument/PhabricatorSearchAbstractDocument.php
+12
-1
src/applications/search/index/abstractdocument/PhabricatorSearchAbstractDocument.php
···
1
1
<?php
2
2
3
3
/*
4
-
* Copyright 2011 Facebook, Inc.
4
+
* Copyright 2012 Facebook, Inc.
5
5
*
6
6
* Licensed under the Apache License, Version 2.0 (the "License");
7
7
* you may not use this file except in compliance with the License.
···
28
28
private $documentModified;
29
29
private $fields = array();
30
30
private $relationships = array();
31
+
32
+
public static function getSupportedTypes() {
33
+
$more = PhabricatorEnv::getEnvConfig('search.more-document-types', array());
34
+
return array(
35
+
PhabricatorPHIDConstants::PHID_TYPE_DREV => 'Differential Revisions',
36
+
PhabricatorPHIDConstants::PHID_TYPE_CMIT => 'Repository Commits',
37
+
PhabricatorPHIDConstants::PHID_TYPE_TASK => 'Maniphest Tasks',
38
+
PhabricatorPHIDConstants::PHID_TYPE_WIKI => 'Phriction Documents',
39
+
PhabricatorPHIDConstants::PHID_TYPE_USER => 'Phabricator Users',
40
+
) + $more;
41
+
}
31
42
32
43
public function setPHID($phid) {
33
44
$this->phid = $phid;
+2
src/applications/search/index/abstractdocument/__init__.php
+2
src/applications/search/index/abstractdocument/__init__.php
···
6
6
7
7
8
8
9
+
phutil_require_module('phabricator', 'applications/phid/constants');
9
10
phutil_require_module('phabricator', 'applications/search/constants/field');
11
+
phutil_require_module('phabricator', 'infrastructure/env');
10
12
11
13
12
14
phutil_require_source('PhabricatorSearchAbstractDocument.php');
+1
-1
src/applications/search/index/indexer/user/PhabricatorSearchUserIndexer.php
+1
-1
src/applications/search/index/indexer/user/PhabricatorSearchUserIndexer.php
···
26
26
$doc = new PhabricatorSearchAbstractDocument();
27
27
$doc->setPHID($user->getPHID());
28
28
$doc->setDocumentType(PhabricatorPHIDConstants::PHID_TYPE_USER);
29
-
$doc->setDocumentTitle($user->getUserName().'('.$user->getRealName().')');
29
+
$doc->setDocumentTitle($user->getUserName().' ('.$user->getRealName().')');
30
30
$doc->setDocumentCreated($user->getDateCreated());
31
31
$doc->setDocumentModified($user->getDateModified());
32
32