@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Provide a page for examining the facts an object generates
Summary:
Depends on D19120. Ref T13083. When you write a fact engine, it's currently somewhat difficult to figure out exactly what it's doing. It would also be difficult to diagnose bugs or report them to the upstream.
To ease this, add a page which shows all the facts an object generates. This allows you to iterate on an engine quickly without needing to reanalyze facts, take a screenshot, easily compare the timeline to the fact view, etc.
Test Plan: Viewed the object fact page for several objects.
Subscribers: yelirekim
Maniphest Tasks: T13083
Differential Revision: https://secure.phabricator.com/D19121
+510
-10
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
2916
2916
'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php',
2917
2917
'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php',
2918
2918
'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php',
2919
+
'PhabricatorFactDatapointQuery' => 'applications/fact/query/PhabricatorFactDatapointQuery.php',
2919
2920
'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php',
2920
2921
'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php',
2921
2922
'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php',
···
2928
2929
'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php',
2929
2930
'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php',
2930
2931
'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php',
2932
+
'PhabricatorFactObjectController' => 'applications/fact/controller/PhabricatorFactObjectController.php',
2931
2933
'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php',
2932
2934
'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php',
2933
2935
'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php',
···
8445
8447
'PhabricatorFactCursor' => 'PhabricatorFactDAO',
8446
8448
'PhabricatorFactDAO' => 'PhabricatorLiskDAO',
8447
8449
'PhabricatorFactDaemon' => 'PhabricatorDaemon',
8450
+
'PhabricatorFactDatapointQuery' => 'Phobject',
8448
8451
'PhabricatorFactDimension' => 'PhabricatorFactDAO',
8449
8452
'PhabricatorFactEngine' => 'Phobject',
8450
8453
'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase',
···
8457
8460
'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow',
8458
8461
'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow',
8459
8462
'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine',
8463
+
'PhabricatorFactObjectController' => 'PhabricatorFactController',
8460
8464
'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension',
8461
8465
'PhabricatorFactRaw' => 'PhabricatorFactDAO',
8462
8466
'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator',
+1
src/applications/fact/application/PhabricatorFactApplication.php
+1
src/applications/fact/application/PhabricatorFactApplication.php
+267
src/applications/fact/controller/PhabricatorFactObjectController.php
+267
src/applications/fact/controller/PhabricatorFactObjectController.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFactObjectController
4
+
extends PhabricatorFactController {
5
+
6
+
public function handleRequest(AphrontRequest $request) {
7
+
$viewer = $request->getViewer();
8
+
9
+
$phid = $request->getURIData('phid');
10
+
$object = id(new PhabricatorObjectQuery())
11
+
->setViewer($viewer)
12
+
->withNames(array($phid))
13
+
->executeOne();
14
+
if (!$object) {
15
+
return new Aphront404Response();
16
+
}
17
+
18
+
$engines = PhabricatorFactEngine::loadAllEngines();
19
+
foreach ($engines as $key => $engine) {
20
+
if (!$engine->supportsDatapointsForObject($object)) {
21
+
unset($engines[$key]);
22
+
}
23
+
}
24
+
25
+
if (!$engines) {
26
+
return $this->newDialog()
27
+
->setTitle(pht('No Engines'))
28
+
->appendParagraph(
29
+
pht(
30
+
'No fact engines support generating facts for this object.'))
31
+
->addCancelButton($this->getApplicationURI());
32
+
}
33
+
34
+
$key_dimension = new PhabricatorFactKeyDimension();
35
+
$object_phid = $object->getPHID();
36
+
37
+
$facts = array();
38
+
$generated_datapoints = array();
39
+
$timings = array();
40
+
foreach ($engines as $key => $engine) {
41
+
$engine_facts = $engine->newFacts();
42
+
$engine_facts = mpull($engine_facts, null, 'getKey');
43
+
$facts[$key] = $engine_facts;
44
+
45
+
$t_start = microtime(true);
46
+
$generated_datapoints[$key] = $engine->newDatapointsForObject($object);
47
+
$t_end = microtime(true);
48
+
49
+
$timings[$key] = ($t_end - $t_start);
50
+
}
51
+
52
+
$object_id = id(new PhabricatorFactObjectDimension())
53
+
->newDimensionID($object_phid, true);
54
+
55
+
$stored_datapoints = id(new PhabricatorFactDatapointQuery())
56
+
->withFacts(array_mergev($facts))
57
+
->withObjectPHIDs(array($object_phid))
58
+
->needVectors(true)
59
+
->execute();
60
+
61
+
$stored_groups = array();
62
+
foreach ($stored_datapoints as $stored_datapoint) {
63
+
$stored_groups[$stored_datapoint['key']][] = $stored_datapoint;
64
+
}
65
+
66
+
$stored_map = array();
67
+
foreach ($engines as $key => $engine) {
68
+
$stored_map[$key] = array();
69
+
foreach ($facts[$key] as $fact) {
70
+
$fact_datapoints = idx($stored_groups, $fact->getKey(), array());
71
+
$fact_datapoints = igroup($fact_datapoints, 'vector');
72
+
$stored_map[$key] += $fact_datapoints;
73
+
}
74
+
}
75
+
76
+
$handle_phids = array();
77
+
$handle_phids[] = $object->getPHID();
78
+
foreach ($generated_datapoints as $key => $datapoint_set) {
79
+
foreach ($datapoint_set as $datapoint) {
80
+
$dimension_phid = $datapoint->getDimensionPHID();
81
+
if ($dimension_phid !== null) {
82
+
$handle_phids[$dimension_phid] = $dimension_phid;
83
+
}
84
+
}
85
+
}
86
+
87
+
foreach ($stored_map as $key => $stored_datapoints) {
88
+
foreach ($stored_datapoints as $vector_key => $datapoints) {
89
+
foreach ($datapoints as $datapoint) {
90
+
$dimension_phid = $datapoint['dimensionPHID'];
91
+
if ($dimension_phid !== null) {
92
+
$handle_phids[$dimension_phid] = $dimension_phid;
93
+
}
94
+
}
95
+
}
96
+
}
97
+
98
+
$handles = $viewer->loadHandles($handle_phids);
99
+
100
+
$dimension_map = id(new PhabricatorFactObjectDimension())
101
+
->newDimensionMap($handle_phids, true);
102
+
103
+
$content = array();
104
+
105
+
$object_list = id(new PHUIPropertyListView())
106
+
->setViewer($viewer)
107
+
->addProperty(
108
+
pht('Object'),
109
+
$handles[$object->getPHID()]->renderLink());
110
+
111
+
$total_cost = array_sum($timings);
112
+
$total_cost = pht('%sms', new PhutilNumber((int)(1000 * $total_cost)));
113
+
$object_list->addProperty(pht('Total Cost'), $total_cost);
114
+
115
+
$object_box = id(new PHUIObjectBoxView())
116
+
->setHeaderText(pht('Fact Extraction Report'))
117
+
->addPropertyList($object_list);
118
+
119
+
$content[] = $object_box;
120
+
121
+
$icon_fact = id(new PHUIIconView())
122
+
->setIcon('fa-line-chart green')
123
+
->setTooltip(pht('Consistent Fact'));
124
+
125
+
$icon_nodata = id(new PHUIIconView())
126
+
->setIcon('fa-question-circle-o violet')
127
+
->setTooltip(pht('No Stored Datapoints'));
128
+
129
+
$icon_new = id(new PHUIIconView())
130
+
->setIcon('fa-plus red')
131
+
->setTooltip(pht('Not Stored'));
132
+
133
+
$icon_surplus = id(new PHUIIconView())
134
+
->setIcon('fa-minus red')
135
+
->setTooltip(pht('Not Generated'));
136
+
137
+
foreach ($engines as $key => $engine) {
138
+
$rows = array();
139
+
foreach ($generated_datapoints[$key] as $datapoint) {
140
+
$dimension_phid = $datapoint->getDimensionPHID();
141
+
if ($dimension_phid !== null) {
142
+
$dimension = $handles[$datapoint->getDimensionPHID()]->renderLink();
143
+
} else {
144
+
$dimension = null;
145
+
}
146
+
147
+
$fact_key = $datapoint->getKey();
148
+
149
+
$fact = idx($facts[$key], $fact_key, null);
150
+
if ($fact) {
151
+
$fact_label = $fact->getName();
152
+
} else {
153
+
$fact_label = $fact_key;
154
+
}
155
+
156
+
$vector_key = $datapoint->newDatapointVector();
157
+
if (isset($stored_map[$key][$vector_key])) {
158
+
unset($stored_map[$key][$vector_key]);
159
+
$icon = $icon_fact;
160
+
} else {
161
+
$icon = $icon_new;
162
+
}
163
+
164
+
$rows[] = array(
165
+
$icon,
166
+
$fact_label,
167
+
$dimension,
168
+
$datapoint->getValue(),
169
+
phabricator_datetime($datapoint->getEpoch(), $viewer),
170
+
);
171
+
}
172
+
173
+
foreach ($stored_map[$key] as $vector_key => $datapoints) {
174
+
foreach ($datapoints as $datapoint) {
175
+
$dimension_phid = $datapoint['dimensionPHID'];
176
+
if ($dimension_phid !== null) {
177
+
$dimension = $handles[$dimension_phid]->renderLink();
178
+
} else {
179
+
$dimension = null;
180
+
}
181
+
182
+
$fact_key = $datapoint['key'];
183
+
$fact = idx($facts[$key], $fact_key, null);
184
+
if ($fact) {
185
+
$fact_label = $fact->getName();
186
+
} else {
187
+
$fact_label = $fact_key;
188
+
}
189
+
190
+
$rows[] = array(
191
+
$icon_surplus,
192
+
$fact_label,
193
+
$dimension,
194
+
$datapoint['value'],
195
+
phabricator_datetime($datapoint['epoch'], $viewer),
196
+
);
197
+
}
198
+
}
199
+
200
+
foreach ($facts[$key] as $fact) {
201
+
$has_any = id(new PhabricatorFactDatapointQuery())
202
+
->withFacts(array($fact))
203
+
->setLimit(1)
204
+
->execute();
205
+
if ($has_any) {
206
+
continue;
207
+
}
208
+
209
+
if (!$has_any) {
210
+
$rows[] = array(
211
+
$icon_nodata,
212
+
$fact->getName(),
213
+
null,
214
+
null,
215
+
null,
216
+
);
217
+
}
218
+
}
219
+
220
+
$table = id(new AphrontTableView($rows))
221
+
->setHeaders(
222
+
array(
223
+
null,
224
+
pht('Fact'),
225
+
pht('Dimension'),
226
+
pht('Value'),
227
+
pht('Date'),
228
+
))
229
+
->setColumnClasses(
230
+
array(
231
+
'',
232
+
'',
233
+
'',
234
+
'n wide right',
235
+
'right',
236
+
));
237
+
238
+
$extraction_cost = $timings[$key];
239
+
$extraction_cost = pht(
240
+
'%sms',
241
+
new PhutilNumber((int)(1000 * $extraction_cost)));
242
+
243
+
$header = pht(
244
+
'%s (%s)',
245
+
get_class($engine),
246
+
$extraction_cost);
247
+
248
+
$box = id(new PHUIObjectBoxView())
249
+
->setHeaderText($header)
250
+
->setTable($table);
251
+
252
+
$content[] = $box;
253
+
}
254
+
255
+
$crumbs = $this->buildApplicationCrumbs()
256
+
->addTextCrumb(pht('Chart'));
257
+
258
+
$title = pht('Chart');
259
+
260
+
return $this->newPage()
261
+
->setTitle($title)
262
+
->setCrumbs($crumbs)
263
+
->appendChild($content);
264
+
265
+
}
266
+
267
+
}
+181
src/applications/fact/query/PhabricatorFactDatapointQuery.php
+181
src/applications/fact/query/PhabricatorFactDatapointQuery.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFactDatapointQuery extends Phobject {
4
+
5
+
private $facts;
6
+
private $objectPHIDs;
7
+
private $limit;
8
+
9
+
private $needVectors;
10
+
11
+
private $keyMap = array();
12
+
private $dimensionMap = array();
13
+
14
+
public function withFacts(array $facts) {
15
+
$this->facts = $facts;
16
+
return $this;
17
+
}
18
+
19
+
public function withObjectPHIDs(array $object_phids) {
20
+
$this->objectPHIDs = $object_phids;
21
+
return $this;
22
+
}
23
+
24
+
public function setLimit($limit) {
25
+
$this->limit = $limit;
26
+
return $this;
27
+
}
28
+
29
+
public function needVectors($need) {
30
+
$this->needVectors = $need;
31
+
return $this;
32
+
}
33
+
34
+
public function execute() {
35
+
$facts = mpull($this->facts, null, 'getKey');
36
+
if (!$facts) {
37
+
throw new Exception(pht('Executing a fact query requires facts.'));
38
+
}
39
+
40
+
$table_map = array();
41
+
foreach ($facts as $fact) {
42
+
$datapoint = $fact->newDatapoint();
43
+
$table = $datapoint->getTableName();
44
+
45
+
if (!isset($table_map[$table])) {
46
+
$table_map[$table] = array(
47
+
'table' => $datapoint,
48
+
'facts' => array(),
49
+
);
50
+
}
51
+
52
+
$table_map[$table]['facts'][] = $fact;
53
+
}
54
+
55
+
$rows = array();
56
+
foreach ($table_map as $spec) {
57
+
$rows[] = $this->executeWithTable($spec);
58
+
}
59
+
$rows = array_mergev($rows);
60
+
61
+
$key_unmap = array_flip($this->keyMap);
62
+
$dimension_unmap = array_flip($this->dimensionMap);
63
+
64
+
$groups = array();
65
+
$need_phids = array();
66
+
foreach ($rows as $row) {
67
+
$groups[$row['keyID']][] = $row;
68
+
69
+
$object_id = $row['objectID'];
70
+
if (!isset($dimension_unmap[$object_id])) {
71
+
$need_phids[$object_id] = $object_id;
72
+
}
73
+
74
+
$dimension_id = $row['dimensionID'];
75
+
if ($dimension_id && !isset($dimension_unmap[$dimension_id])) {
76
+
$need_phids[$dimension_id] = $dimension_id;
77
+
}
78
+
}
79
+
80
+
$dimension_unmap += id(new PhabricatorFactObjectDimension())
81
+
->newDimensionUnmap($need_phids);
82
+
83
+
$results = array();
84
+
foreach ($groups as $key_id => $rows) {
85
+
$key = $key_unmap[$key_id];
86
+
$fact = $facts[$key];
87
+
$datapoint = $fact->newDatapoint();
88
+
foreach ($rows as $row) {
89
+
$dimension_id = $row['dimensionID'];
90
+
if ($dimension_id) {
91
+
if (!isset($dimension_unmap[$dimension_id])) {
92
+
continue;
93
+
} else {
94
+
$dimension_phid = $dimension_unmap[$dimension_id];
95
+
}
96
+
} else {
97
+
$dimension_phid = null;
98
+
}
99
+
100
+
$object_id = $row['objectID'];
101
+
if (!isset($dimension_unmap[$object_id])) {
102
+
continue;
103
+
} else {
104
+
$object_phid = $dimension_unmap[$object_id];
105
+
}
106
+
107
+
$result = array(
108
+
'key' => $key,
109
+
'objectPHID' => $object_phid,
110
+
'dimensionPHID' => $dimension_phid,
111
+
'value' => (int)$row['value'],
112
+
'epoch' => $row['epoch'],
113
+
);
114
+
115
+
if ($this->needVectors) {
116
+
$result['vector'] = $datapoint->newRawVector($result);
117
+
}
118
+
119
+
$results[] = $result;
120
+
}
121
+
}
122
+
123
+
return $results;
124
+
}
125
+
126
+
private function executeWithTable(array $spec) {
127
+
$table = $spec['table'];
128
+
$facts = $spec['facts'];
129
+
$conn = $table->establishConnection('r');
130
+
131
+
$fact_keys = mpull($facts, 'getKey');
132
+
$this->keyMap = id(new PhabricatorFactKeyDimension())
133
+
->newDimensionMap($fact_keys);
134
+
135
+
if (!$this->keyMap) {
136
+
return array();
137
+
}
138
+
139
+
$where = array();
140
+
141
+
$where[] = qsprintf(
142
+
$conn,
143
+
'keyID IN (%Ld)',
144
+
$this->keyMap);
145
+
146
+
if ($this->objectPHIDs) {
147
+
$object_map = id(new PhabricatorFactObjectDimension())
148
+
->newDimensionMap($this->objectPHIDs);
149
+
if (!$object_map) {
150
+
return array();
151
+
}
152
+
153
+
$this->dimensionMap = $object_map;
154
+
155
+
$where[] = qsprintf(
156
+
$conn,
157
+
'objectID IN (%Ld)',
158
+
$this->dimensionMap);
159
+
}
160
+
161
+
$where = '('.implode(') AND (', $where).')';
162
+
163
+
if ($this->limit) {
164
+
$limit = qsprintf(
165
+
$conn,
166
+
'LIMIT %d',
167
+
$this->limit);
168
+
} else {
169
+
$limit = '';
170
+
}
171
+
172
+
return queryfx_all(
173
+
$conn,
174
+
'SELECT keyID, objectID, dimensionID, value, epoch
175
+
FROM %T WHERE %Q %Q',
176
+
$table->getTableName(),
177
+
$where,
178
+
$limit);
179
+
}
180
+
181
+
}
+31
-10
src/applications/fact/storage/PhabricatorFactDimension.php
+31
-10
src/applications/fact/storage/PhabricatorFactDimension.php
···
4
4
5
5
abstract protected function getDimensionColumnName();
6
6
7
-
final public function newDimensionID($key) {
8
-
$map = $this->newDimensionMap(array($key));
7
+
final public function newDimensionID($key, $create = false) {
8
+
$map = $this->newDimensionMap(array($key), $create);
9
9
return idx($map, $key);
10
+
}
11
+
12
+
final public function newDimensionUnmap(array $ids) {
13
+
if (!$ids) {
14
+
return array();
15
+
}
16
+
17
+
$conn = $this->establishConnection('r');
18
+
$column = $this->getDimensionColumnName();
19
+
20
+
$rows = queryfx_all(
21
+
$conn,
22
+
'SELECT id, %C FROM %T WHERE id IN (%Ld)',
23
+
$column,
24
+
$this->getTableName(),
25
+
$ids);
26
+
$rows = ipull($rows, $column, 'id');
27
+
28
+
return $rows;
10
29
}
11
30
12
31
final public function newDimensionMap(array $keys, $create = false) {
···
52
71
$key);
53
72
}
54
73
55
-
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
56
-
queryfx(
57
-
$conn,
58
-
'INSERT IGNORE INTO %T (%C) VALUES %Q',
59
-
$this->getTableName(),
60
-
$column,
61
-
$chunk);
62
-
}
74
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
75
+
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
76
+
queryfx(
77
+
$conn,
78
+
'INSERT IGNORE INTO %T (%C) VALUES %Q',
79
+
$this->getTableName(),
80
+
$column,
81
+
$chunk);
82
+
}
83
+
unset($unguarded);
63
84
64
85
$rows = queryfx_all(
65
86
$conn,
+26
src/applications/fact/storage/PhabricatorFactIntDatapoint.php
+26
src/applications/fact/storage/PhabricatorFactIntDatapoint.php
···
58
58
return $this->dimensionPHID;
59
59
}
60
60
61
+
public function newDatapointVector() {
62
+
return $this->formatVector(
63
+
array(
64
+
$this->key,
65
+
$this->objectPHID,
66
+
$this->dimensionPHID,
67
+
$this->value,
68
+
$this->epoch,
69
+
));
70
+
}
71
+
72
+
public function newRawVector(array $spec) {
73
+
return $this->formatVector(
74
+
array(
75
+
$spec['key'],
76
+
$spec['objectPHID'],
77
+
$spec['dimensionPHID'],
78
+
$spec['value'],
79
+
$spec['epoch'],
80
+
));
81
+
}
82
+
83
+
private function formatVector(array $vector) {
84
+
return implode(':', $vector);
85
+
}
86
+
61
87
}