@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Improve Conduit performance for custom fields
Summary:
Ref T11404. Depends on D16350.
Currently, custom fields can issue "N+1" queries in some cases, so querying 100 revisions issues 100 extra queries.
This affects all `*.search` endpoints for objects with custom fields, and some older endpoints (notably `differential.query`).
This change bulk loads "normal" custom fields, which gets rid of some of these queries. Instead of loading fields for each object, we build a big list of all fields and load them all at once.
The next change will tackle the remaining inefficient edge queries.
Test Plan:
- Configured a custom field with normal database storage in Differential.
- Ran `differential.query`, looking at custom fields in results for correctness.
- Ran `differential.revision.search`, looking at custom fields in results for correctness.
- In both cases, observed queries drop from `3N` to `2N` (all the "normal" custom field stuff got bulk loaded).
Reviewers: yelirekim, chad
Reviewed By: chad
Maniphest Tasks: T11404
Differential Revision: https://secure.phabricator.com/D16351
+223
-54
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
2258
2258
'PhabricatorCustomFieldNumericIndexStorage' => 'infrastructure/customfield/storage/PhabricatorCustomFieldNumericIndexStorage.php',
2259
2259
'PhabricatorCustomFieldSearchEngineExtension' => 'infrastructure/customfield/engineextension/PhabricatorCustomFieldSearchEngineExtension.php',
2260
2260
'PhabricatorCustomFieldStorage' => 'infrastructure/customfield/storage/PhabricatorCustomFieldStorage.php',
2261
+
'PhabricatorCustomFieldStorageQuery' => 'infrastructure/customfield/query/PhabricatorCustomFieldStorageQuery.php',
2261
2262
'PhabricatorCustomFieldStringIndexStorage' => 'infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php',
2262
2263
'PhabricatorCustomHeaderConfigType' => 'applications/config/custom/PhabricatorCustomHeaderConfigType.php',
2263
2264
'PhabricatorDaemon' => 'infrastructure/daemon/PhabricatorDaemon.php',
···
6994
6995
'PhabricatorCustomFieldNumericIndexStorage' => 'PhabricatorCustomFieldIndexStorage',
6995
6996
'PhabricatorCustomFieldSearchEngineExtension' => 'PhabricatorSearchEngineExtension',
6996
6997
'PhabricatorCustomFieldStorage' => 'PhabricatorLiskDAO',
6998
+
'PhabricatorCustomFieldStorageQuery' => 'Phobject',
6997
6999
'PhabricatorCustomFieldStringIndexStorage' => 'PhabricatorCustomFieldIndexStorage',
6998
7000
'PhabricatorCustomHeaderConfigType' => 'PhabricatorConfigOptionType',
6999
7001
'PhabricatorDaemon' => 'PhutilDaemon',
+21
-4
src/applications/differential/conduit/DifferentialConduitAPIMethod.php
+21
-4
src/applications/differential/conduit/DifferentialConduitAPIMethod.php
···
150
150
array $revisions) {
151
151
assert_instances_of($revisions, 'DifferentialRevision');
152
152
153
-
$results = array();
153
+
$field_lists = array();
154
154
foreach ($revisions as $revision) {
155
-
// TODO: This is inefficient and issues a query for each object.
155
+
$revision_phid = $revision->getPHID();
156
+
156
157
$field_list = PhabricatorCustomField::getObjectFields(
157
158
$revision,
158
159
PhabricatorCustomField::ROLE_CONDUIT);
159
160
160
161
$field_list
161
162
->setViewer($viewer)
162
-
->readFieldsFromStorage($revision);
163
+
->readFieldsFromObject($revision);
164
+
165
+
$field_lists[$revision_phid] = $field_list;
166
+
}
167
+
168
+
$all_fields = array();
169
+
foreach ($field_lists as $field_list) {
170
+
foreach ($field_list->getFields() as $field) {
171
+
$all_fields[] = $field;
172
+
}
173
+
}
163
174
175
+
id(new PhabricatorCustomFieldStorageQuery())
176
+
->addFields($all_fields)
177
+
->execute();
178
+
179
+
$results = array();
180
+
foreach ($field_lists as $revision_phid => $field_list) {
164
181
foreach ($field_list->getFields() as $field) {
165
182
$field_key = $field->getFieldKeyForConduit();
166
183
$value = $field->getConduitDictionaryValue();
167
-
$results[$revision->getPHID()][$field_key] = $value;
184
+
$results[$revision_phid][$field_key] = $value;
168
185
}
169
186
}
170
187
+34
-9
src/infrastructure/customfield/engineextension/PhabricatorCustomFieldSearchEngineExtension.php
+34
-9
src/infrastructure/customfield/engineextension/PhabricatorCustomFieldSearchEngineExtension.php
···
80
80
return $map;
81
81
}
82
82
83
-
public function getFieldValuesForConduit($object, $data) {
84
-
// TODO: This is currently very inefficient. We should bulk-load these
85
-
// field values instead.
83
+
public function loadExtensionConduitData(array $objects) {
84
+
$viewer = $this->getViewer();
85
+
86
+
$field_map = array();
87
+
foreach ($objects as $object) {
88
+
$object_phid = $object->getPHID();
89
+
90
+
$fields = PhabricatorCustomField::getObjectFields(
91
+
$object,
92
+
PhabricatorCustomField::ROLE_CONDUIT);
93
+
94
+
$fields
95
+
->setViewer($viewer)
96
+
->readFieldsFromObject($object);
97
+
98
+
$field_map[$object_phid] = $fields;
99
+
}
100
+
101
+
$all_fields = array();
102
+
foreach ($field_map as $field_list) {
103
+
foreach ($field_list->getFields() as $field) {
104
+
$all_fields[] = $field;
105
+
}
106
+
}
107
+
108
+
id(new PhabricatorCustomFieldStorageQuery())
109
+
->addFields($all_fields)
110
+
->execute();
86
111
87
-
$fields = PhabricatorCustomField::getObjectFields(
88
-
$object,
89
-
PhabricatorCustomField::ROLE_CONDUIT);
112
+
return array(
113
+
'fields' => $field_map,
114
+
);
115
+
}
90
116
91
-
$fields
92
-
->setViewer($this->getViewer())
93
-
->readFieldsFromStorage($object);
117
+
public function getFieldValuesForConduit($object, $data) {
118
+
$fields = $data['fields'][$object->getPHID()];
94
119
95
120
$map = array();
96
121
foreach ($fields->getFields() as $field) {
+18
-41
src/infrastructure/customfield/field/PhabricatorCustomFieldList.php
+18
-41
src/infrastructure/customfield/field/PhabricatorCustomFieldList.php
···
29
29
return $this;
30
30
}
31
31
32
+
public function readFieldsFromObject(
33
+
PhabricatorCustomFieldInterface $object) {
34
+
35
+
$fields = $this->getFields();
36
+
37
+
foreach ($fields as $field) {
38
+
$field
39
+
->setObject($object)
40
+
->readValueFromObject($object);
41
+
}
42
+
43
+
return $this;
44
+
}
32
45
33
46
/**
34
47
* Read stored values for all fields which support storage.
···
39
52
public function readFieldsFromStorage(
40
53
PhabricatorCustomFieldInterface $object) {
41
54
42
-
foreach ($this->fields as $field) {
43
-
$field->setObject($object);
44
-
$field->readValueFromObject($object);
45
-
}
55
+
$this->readFieldsFromObject($object);
46
56
47
-
$keys = array();
48
-
foreach ($this->fields as $field) {
49
-
if ($field->shouldEnableForRole(PhabricatorCustomField::ROLE_STORAGE)) {
50
-
$keys[$field->getFieldIndex()] = $field;
51
-
}
52
-
}
53
-
54
-
if (!$keys) {
55
-
return $this;
56
-
}
57
-
58
-
// NOTE: We assume all fields share the same storage. This isn't guaranteed
59
-
// to be true, but always is for now.
60
-
61
-
$table = head($keys)->newStorageObject();
62
-
63
-
$objects = array();
64
-
if ($object->getPHID()) {
65
-
$objects = $table->loadAllWhere(
66
-
'objectPHID = %s AND fieldIndex IN (%Ls)',
67
-
$object->getPHID(),
68
-
array_keys($keys));
69
-
$objects = mpull($objects, null, 'getFieldIndex');
70
-
}
71
-
72
-
foreach ($keys as $key => $field) {
73
-
$storage = idx($objects, $key);
74
-
if ($storage) {
75
-
$field->setValueFromStorage($storage->getFieldValue());
76
-
$field->didSetValueFromStorage();
77
-
} else if ($object->getPHID()) {
78
-
// NOTE: We set this only if the object exists. Otherwise, we allow the
79
-
// field to retain any default value it may have.
80
-
$field->setValueFromStorage(null);
81
-
$field->didSetValueFromStorage();
82
-
}
83
-
}
57
+
$fields = $this->getFields();
58
+
id(new PhabricatorCustomFieldStorageQuery())
59
+
->addFields($fields)
60
+
->execute();
84
61
85
62
return $this;
86
63
}
+84
src/infrastructure/customfield/query/PhabricatorCustomFieldStorageQuery.php
+84
src/infrastructure/customfield/query/PhabricatorCustomFieldStorageQuery.php
···
1
+
<?php
2
+
3
+
/**
4
+
* Load custom field data from storage.
5
+
*
6
+
* This query loads the data directly into the field objects and does not
7
+
* return it to the caller. It can bulk load data for any list of fields,
8
+
* even if they have different objects or object types.
9
+
*/
10
+
final class PhabricatorCustomFieldStorageQuery extends Phobject {
11
+
12
+
private $fieldMap = array();
13
+
private $storageSources = array();
14
+
15
+
public function addFields(array $fields) {
16
+
assert_instances_of($fields, 'PhabricatorCustomField');
17
+
18
+
foreach ($fields as $field) {
19
+
$this->addField($field);
20
+
}
21
+
22
+
return $this;
23
+
}
24
+
25
+
public function addField(PhabricatorCustomField $field) {
26
+
$role_storage = PhabricatorCustomField::ROLE_STORAGE;
27
+
28
+
if (!$field->shouldEnableForRole($role_storage)) {
29
+
return $this;
30
+
}
31
+
32
+
$storage = $field->newStorageObject();
33
+
$source_key = $storage->getStorageSourceKey();
34
+
35
+
$this->fieldMap[$source_key][] = $field;
36
+
37
+
if (empty($this->storageSources[$source_key])) {
38
+
$this->storageSources[$source_key] = $storage;
39
+
}
40
+
41
+
return $this;
42
+
}
43
+
44
+
public function execute() {
45
+
foreach ($this->storageSources as $source_key => $storage) {
46
+
$fields = idx($this->fieldMap, $source_key, array());
47
+
$this->loadFieldsFromStorage($storage, $fields);
48
+
}
49
+
}
50
+
51
+
private function loadFieldsFromStorage($storage, array $fields) {
52
+
// Only try to load fields which have a persisted object.
53
+
$loadable = array();
54
+
foreach ($fields as $key => $field) {
55
+
$object = $field->getObject();
56
+
$phid = $object->getPHID();
57
+
if (!$phid) {
58
+
continue;
59
+
}
60
+
61
+
$loadable[$key] = $field;
62
+
}
63
+
64
+
if ($loadable) {
65
+
$data = $storage->loadStorageSourceData($loadable);
66
+
} else {
67
+
$data = array();
68
+
}
69
+
70
+
foreach ($fields as $key => $field) {
71
+
if (array_key_exists($key, $data)) {
72
+
$value = $data[$key];
73
+
$field->setValueFromStorage($value);
74
+
$field->didSetValueFromStorage();
75
+
} else if (isset($loadable[$key])) {
76
+
// NOTE: We set this only if the object exists. Otherwise, we allow
77
+
// the field to retain any default value it may have.
78
+
$field->setValueFromStorage(null);
79
+
$field->didSetValueFromStorage();
80
+
}
81
+
}
82
+
}
83
+
84
+
}
+64
src/infrastructure/customfield/storage/PhabricatorCustomFieldStorage.php
+64
src/infrastructure/customfield/storage/PhabricatorCustomFieldStorage.php
···
23
23
) + parent::getConfiguration();
24
24
}
25
25
26
+
27
+
/**
28
+
* Get a key which uniquely identifies this storage source.
29
+
*
30
+
* When loading custom fields, fields using sources with the same source key
31
+
* are loaded in bulk.
32
+
*
33
+
* @return string Source identifier.
34
+
*/
35
+
final public function getStorageSourceKey() {
36
+
return $this->getApplicationName().'/'.$this->getTableName();
37
+
}
38
+
39
+
40
+
/**
41
+
* Load stored data for custom fields.
42
+
*
43
+
* Given a map of fields, return a map with any stored data for those fields.
44
+
* The keys in the result should correspond to the keys in the input. The
45
+
* fields in the list may belong to different objects.
46
+
*
47
+
* @param map<string, PhabricatorCustomField> Map of fields.
48
+
* @return map<String, PhabricatorCustomField> Map of available field data.
49
+
*/
50
+
final public function loadStorageSourceData(array $fields) {
51
+
$map = array();
52
+
$indexes = array();
53
+
$object_phids = array();
54
+
55
+
foreach ($fields as $key => $field) {
56
+
$index = $field->getFieldIndex();
57
+
$object_phid = $field->getObject()->getPHID();
58
+
59
+
$map[$index][$object_phid] = $key;
60
+
$indexes[$index] = $index;
61
+
$object_phids[$object_phid] = $object_phid;
62
+
}
63
+
64
+
if (!$indexes) {
65
+
return array();
66
+
}
67
+
68
+
$conn = $this->establishConnection('r');
69
+
$rows = queryfx_all(
70
+
$conn,
71
+
'SELECT objectPHID, fieldIndex, fieldValue FROM %T
72
+
WHERE objectPHID IN (%Ls) AND fieldIndex IN (%Ls)',
73
+
$this->getTableName(),
74
+
$object_phids,
75
+
$indexes);
76
+
77
+
$result = array();
78
+
foreach ($rows as $row) {
79
+
$index = $row['fieldIndex'];
80
+
$object_phid = $row['objectPHID'];
81
+
$value = $row['fieldValue'];
82
+
83
+
$key = $map[$index][$object_phid];
84
+
$result[$key] = $value;
85
+
}
86
+
87
+
return $result;
88
+
}
89
+
26
90
}