@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make Facts more modern, DRY, and dimensional
Summary:
Ref T13083. Facts has a fair amount of weird hardcoding and duplication of responsibilities. Reduce this somewhat: no more hard-coded fact aggregates, no more database-driven list of available facts, etc. Generally, derive all objective truth from FactEngines. This is more similar to how most other modern applications work.
For clarity, hopefully: rename "FactSpec" to "Fact". Rename "RawFact" to "Datapoint".
Split the fairly optimistic "RawFact" table into an "IntDatapoint" table with less stuff in it, then dimension tables for the object PHIDs and key names. This is primarily aimed at reducing the row size of each datapoint. At the time I originally wrote this code we hadn't experimented much with storing similar data in multiple tables, but this is now more common and has worked well elsewhere (CustomFields, Edges, Ferret) so I don't anticipate this causing issues. If we need more complex or multidimension/multivalue tables later we can accommodate them. The queries a single table supports (like "all facts of all kinds in some time window") don't make any sense as far as I can tell and could likely be UNION ALL'd anyway.
Remove all the aggregation stuff for now, it's not really clear to me what this should look like.
Test Plan: Ran `bin/fact analyze` and viewed web UI. Nothing exploded too violently.
Subscribers: yelirekim
Maniphest Tasks: T13083
Differential Revision: https://secure.phabricator.com/D19119
+422
-467
+5
resources/sql/autopatches/20180218.fact.01.dim.key.sql
+5
resources/sql/autopatches/20180218.fact.01.dim.key.sql
+5
resources/sql/autopatches/20180218.fact.02.dim.obj.sql
+5
resources/sql/autopatches/20180218.fact.02.dim.obj.sql
+8
resources/sql/autopatches/20180218.fact.03.data.int.sql
+8
resources/sql/autopatches/20180218.fact.03.data.int.sql
···
1
+
CREATE TABLE {$NAMESPACE}_fact.fact_intdatapoint (
2
+
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
keyID INT UNSIGNED NOT NULL,
4
+
objectID INT UNSIGNED NOT NULL,
5
+
dimensionID INT UNSIGNED,
6
+
value BIGINT SIGNED NOT NULL,
7
+
epoch INT UNSIGNED NOT NULL
8
+
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
+14
-10
src/__phutil_library_map__.php
+14
-10
src/__phutil_library_map__.php
···
2907
2907
'PhabricatorExternalAccountsSettingsPanel' => 'applications/settings/panel/PhabricatorExternalAccountsSettingsPanel.php',
2908
2908
'PhabricatorExtraConfigSetupCheck' => 'applications/config/check/PhabricatorExtraConfigSetupCheck.php',
2909
2909
'PhabricatorFacebookAuthProvider' => 'applications/auth/provider/PhabricatorFacebookAuthProvider.php',
2910
+
'PhabricatorFact' => 'applications/fact/fact/PhabricatorFact.php',
2910
2911
'PhabricatorFactAggregate' => 'applications/fact/storage/PhabricatorFactAggregate.php',
2911
2912
'PhabricatorFactApplication' => 'applications/fact/application/PhabricatorFactApplication.php',
2912
2913
'PhabricatorFactChartController' => 'applications/fact/controller/PhabricatorFactChartController.php',
2913
2914
'PhabricatorFactController' => 'applications/fact/controller/PhabricatorFactController.php',
2914
-
'PhabricatorFactCountEngine' => 'applications/fact/engine/PhabricatorFactCountEngine.php',
2915
2915
'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php',
2916
2916
'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php',
2917
2917
'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php',
2918
+
'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php',
2918
2919
'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php',
2919
2920
'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php',
2920
2921
'PhabricatorFactHomeController' => 'applications/fact/controller/PhabricatorFactHomeController.php',
2921
-
'PhabricatorFactLastUpdatedEngine' => 'applications/fact/engine/PhabricatorFactLastUpdatedEngine.php',
2922
+
'PhabricatorFactIntDatapoint' => 'applications/fact/storage/PhabricatorFactIntDatapoint.php',
2923
+
'PhabricatorFactKeyDimension' => 'applications/fact/storage/PhabricatorFactKeyDimension.php',
2922
2924
'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php',
2923
2925
'PhabricatorFactManagementCursorsWorkflow' => 'applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php',
2924
2926
'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php',
2925
2927
'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php',
2926
-
'PhabricatorFactManagementStatusWorkflow' => 'applications/fact/management/PhabricatorFactManagementStatusWorkflow.php',
2927
2928
'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php',
2929
+
'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php',
2930
+
'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php',
2928
2931
'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php',
2929
-
'PhabricatorFactSimpleSpec' => 'applications/fact/spec/PhabricatorFactSimpleSpec.php',
2930
-
'PhabricatorFactSpec' => 'applications/fact/spec/PhabricatorFactSpec.php',
2931
2932
'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php',
2932
2933
'PhabricatorFavoritesApplication' => 'applications/favorites/application/PhabricatorFavoritesApplication.php',
2933
2934
'PhabricatorFavoritesController' => 'applications/favorites/controller/PhabricatorFavoritesController.php',
···
3716
3717
'PhabricatorPirateEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorPirateEnglishTranslation.php',
3717
3718
'PhabricatorPlatformSite' => 'aphront/site/PhabricatorPlatformSite.php',
3718
3719
'PhabricatorPointsEditField' => 'applications/transactions/editfield/PhabricatorPointsEditField.php',
3720
+
'PhabricatorPointsFact' => 'applications/fact/fact/PhabricatorPointsFact.php',
3719
3721
'PhabricatorPolicies' => 'applications/policy/constants/PhabricatorPolicies.php',
3720
3722
'PhabricatorPolicy' => 'applications/policy/storage/PhabricatorPolicy.php',
3721
3723
'PhabricatorPolicyApplication' => 'applications/policy/application/PhabricatorPolicyApplication.php',
···
8433
8435
'PhabricatorExternalAccountsSettingsPanel' => 'PhabricatorSettingsPanel',
8434
8436
'PhabricatorExtraConfigSetupCheck' => 'PhabricatorSetupCheck',
8435
8437
'PhabricatorFacebookAuthProvider' => 'PhabricatorOAuth2AuthProvider',
8438
+
'PhabricatorFact' => 'Phobject',
8436
8439
'PhabricatorFactAggregate' => 'PhabricatorFactDAO',
8437
8440
'PhabricatorFactApplication' => 'PhabricatorApplication',
8438
8441
'PhabricatorFactChartController' => 'PhabricatorFactController',
8439
8442
'PhabricatorFactController' => 'PhabricatorController',
8440
-
'PhabricatorFactCountEngine' => 'PhabricatorFactEngine',
8441
8443
'PhabricatorFactCursor' => 'PhabricatorFactDAO',
8442
8444
'PhabricatorFactDAO' => 'PhabricatorLiskDAO',
8443
8445
'PhabricatorFactDaemon' => 'PhabricatorDaemon',
8446
+
'PhabricatorFactDimension' => 'PhabricatorFactDAO',
8444
8447
'PhabricatorFactEngine' => 'Phobject',
8445
8448
'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase',
8446
8449
'PhabricatorFactHomeController' => 'PhabricatorFactController',
8447
-
'PhabricatorFactLastUpdatedEngine' => 'PhabricatorFactEngine',
8450
+
'PhabricatorFactIntDatapoint' => 'PhabricatorFactDAO',
8451
+
'PhabricatorFactKeyDimension' => 'PhabricatorFactDimension',
8448
8452
'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow',
8449
8453
'PhabricatorFactManagementCursorsWorkflow' => 'PhabricatorFactManagementWorkflow',
8450
8454
'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow',
8451
8455
'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow',
8452
-
'PhabricatorFactManagementStatusWorkflow' => 'PhabricatorFactManagementWorkflow',
8453
8456
'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow',
8457
+
'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine',
8458
+
'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension',
8454
8459
'PhabricatorFactRaw' => 'PhabricatorFactDAO',
8455
-
'PhabricatorFactSimpleSpec' => 'PhabricatorFactSpec',
8456
-
'PhabricatorFactSpec' => 'Phobject',
8457
8460
'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator',
8458
8461
'PhabricatorFavoritesApplication' => 'PhabricatorApplication',
8459
8462
'PhabricatorFavoritesController' => 'PhabricatorController',
···
9373
9376
'PhabricatorPirateEnglishTranslation' => 'PhutilTranslation',
9374
9377
'PhabricatorPlatformSite' => 'PhabricatorSite',
9375
9378
'PhabricatorPointsEditField' => 'PhabricatorEditField',
9379
+
'PhabricatorPointsFact' => 'PhabricatorFact',
9376
9380
'PhabricatorPolicies' => 'PhabricatorPolicyConstants',
9377
9381
'PhabricatorPolicy' => array(
9378
9382
'PhabricatorPolicyDAO',
-6
src/applications/differential/application/PhabricatorDifferentialApplication.php
-6
src/applications/differential/application/PhabricatorDifferentialApplication.php
-6
src/applications/diffusion/application/PhabricatorDiffusionApplication.php
-6
src/applications/diffusion/application/PhabricatorDiffusionApplication.php
+17
-12
src/applications/fact/controller/PhabricatorFactChartController.php
+17
-12
src/applications/fact/controller/PhabricatorFactChartController.php
···
5
5
public function handleRequest(AphrontRequest $request) {
6
6
$viewer = $request->getViewer();
7
7
8
-
$table = new PhabricatorFactRaw();
8
+
$series = $request->getStr('y1');
9
+
10
+
$facts = PhabricatorFact::getAllFacts();
11
+
$fact = idx($facts, $series);
12
+
13
+
if (!$fact) {
14
+
return new Aphront404Response();
15
+
}
16
+
17
+
$key_id = id(new PhabricatorFactKeyDimension())
18
+
->newDimensionID($fact->getKey());
19
+
20
+
$table = $fact->newDatapoint();
9
21
$conn_r = $table->establishConnection('r');
10
22
$table_name = $table->getTableName();
11
23
12
-
$series = $request->getStr('y1');
13
-
14
-
$specs = PhabricatorFactSpec::newSpecsForFactTypes(
15
-
PhabricatorFactEngine::loadAllEngines(),
16
-
array($series));
17
-
$spec = idx($specs, $series);
18
-
19
24
$data = queryfx_all(
20
25
$conn_r,
21
-
'SELECT valueX, epoch FROM %T WHERE factType = %s ORDER BY epoch ASC',
26
+
'SELECT value, epoch FROM %T WHERE keyID = %d ORDER BY epoch ASC',
22
27
$table_name,
23
-
$series);
28
+
$key_id);
24
29
25
30
$points = array();
26
31
$sum = 0;
27
32
foreach ($data as $key => $row) {
28
-
$sum += (int)$row['valueX'];
33
+
$sum += (int)$row['value'];
29
34
$points[(int)$row['epoch']] = $sum;
30
35
}
31
36
···
71
76
));
72
77
73
78
$box = id(new PHUIObjectBoxView())
74
-
->setHeaderText(pht('Count of %s', $spec->getName()))
79
+
->setHeaderText(pht('Count of %s', $fact->getName()))
75
80
->appendChild($chart);
76
81
77
82
$crumbs = $this->buildApplicationCrumbs();
+6
-73
src/applications/fact/controller/PhabricatorFactHomeController.php
+6
-73
src/applications/fact/controller/PhabricatorFactHomeController.php
···
15
15
return id(new AphrontRedirectResponse())->setURI($uri);
16
16
}
17
17
18
-
$types = array(
19
-
'+N:*',
20
-
'+N:DREV',
21
-
'updated',
22
-
);
23
-
24
-
$engines = PhabricatorFactEngine::loadAllEngines();
25
-
$specs = PhabricatorFactSpec::newSpecsForFactTypes($engines, $types);
26
-
27
-
$facts = id(new PhabricatorFactAggregate())->loadAllWhere(
28
-
'factType IN (%Ls)',
29
-
$types);
30
-
31
-
$rows = array();
32
-
foreach ($facts as $fact) {
33
-
$spec = $specs[$fact->getFactType()];
34
-
35
-
$name = $spec->getName();
36
-
$value = $spec->formatValueForDisplay($viewer, $fact->getValueX());
37
-
38
-
$rows[] = array($name, $value);
39
-
}
40
-
41
-
$table = new AphrontTableView($rows);
42
-
$table->setHeaders(
43
-
array(
44
-
pht('Fact'),
45
-
pht('Value'),
46
-
));
47
-
$table->setColumnClasses(
48
-
array(
49
-
'wide',
50
-
'n',
51
-
));
52
-
53
-
$panel = new PHUIObjectBoxView();
54
-
$panel->setHeaderText(pht('Facts'));
55
-
$panel->setTable($table);
56
-
57
18
$chart_form = $this->buildChartForm();
58
19
59
20
$crumbs = $this->buildApplicationCrumbs();
···
64
25
return $this->newPage()
65
26
->setTitle($title)
66
27
->setCrumbs($crumbs)
67
-
->appendChild(array(
68
-
$chart_form,
69
-
$panel,
70
-
));
71
-
28
+
->appendChild(
29
+
array(
30
+
$chart_form,
31
+
));
72
32
}
73
33
74
34
private function buildChartForm() {
75
35
$request = $this->getRequest();
76
36
$viewer = $request->getUser();
77
37
78
-
$table = new PhabricatorFactRaw();
79
-
$conn_r = $table->establishConnection('r');
80
-
$table_name = $table->getTableName();
81
-
82
-
$facts = queryfx_all(
83
-
$conn_r,
84
-
'SELECT DISTINCT factType from %T',
85
-
$table_name);
86
-
87
-
$specs = PhabricatorFactSpec::newSpecsForFactTypes(
88
-
PhabricatorFactEngine::loadAllEngines(),
89
-
ipull($facts, 'factType'));
90
-
91
-
$options = array();
92
-
foreach ($specs as $spec) {
93
-
if ($spec->getUnit() == PhabricatorFactSpec::UNIT_COUNT) {
94
-
$options[$spec->getType()] = $spec->getName();
95
-
}
96
-
}
97
-
98
-
if (!$options) {
99
-
return id(new PHUIInfoView())
100
-
->setSeverity(PHUIInfoView::SEVERITY_NODATA)
101
-
->setTitle(pht('No Chartable Facts'))
102
-
->appendChild(phutil_tag(
103
-
'p',
104
-
array(),
105
-
pht('There are no facts that can be plotted yet.')));
106
-
}
38
+
$specs = PhabricatorFact::getAllFacts();
39
+
$options = mpull($specs, 'getName', 'getKey');
107
40
108
41
$form = id(new AphrontFormView())
109
42
->setUser($viewer)
+58
-76
src/applications/fact/daemon/PhabricatorFactDaemon.php
+58
-76
src/applications/fact/daemon/PhabricatorFactDaemon.php
···
4
4
5
5
private $engines;
6
6
7
-
const RAW_FACT_BUFFER_LIMIT = 128;
8
-
9
7
protected function run() {
10
8
$this->setEngines(PhabricatorFactEngine::loadAllEngines());
11
9
while (!$this->shouldExit()) {
···
15
13
foreach ($iterators as $iterator_name => $iterator) {
16
14
$this->processIteratorWithCursor($iterator_name, $iterator);
17
15
}
18
-
$this->processAggregates();
19
16
20
17
$this->log(pht('Zzz...'));
21
18
$this->sleep(60 * 5);
···
72
69
public function processIterator($iterator) {
73
70
$result = null;
74
71
75
-
$raw_facts = array();
72
+
$datapoints = array();
76
73
foreach ($iterator as $key => $object) {
77
74
$phid = $object->getPHID();
78
75
$this->log(pht('Processing %s...', $phid));
79
-
$raw_facts[$phid] = $this->computeRawFacts($object);
80
-
if (count($raw_facts) > self::RAW_FACT_BUFFER_LIMIT) {
81
-
$this->updateRawFacts($raw_facts);
82
-
$raw_facts = array();
76
+
$datapoints[$phid] = $this->newDatapoints($object);
77
+
if (count($datapoints) > 1024) {
78
+
$this->updateDatapoints($datapoints);
79
+
$datapoints = array();
83
80
}
84
81
$result = $key;
85
82
}
86
83
87
-
if ($raw_facts) {
88
-
$this->updateRawFacts($raw_facts);
89
-
$raw_facts = array();
84
+
if ($datapoints) {
85
+
$this->updateDatapoints($datapoints);
86
+
$datapoints = array();
90
87
}
91
88
92
89
return $result;
93
90
}
94
91
95
-
public function processAggregates() {
96
-
$this->log(pht('Processing aggregates.'));
97
-
98
-
$facts = $this->computeAggregateFacts();
99
-
$this->updateAggregateFacts($facts);
100
-
}
101
-
102
-
private function computeAggregateFacts() {
92
+
private function newDatapoints(PhabricatorLiskDAO $object) {
103
93
$facts = array();
104
94
foreach ($this->engines as $engine) {
105
-
if (!$engine->shouldComputeAggregateFacts()) {
95
+
if (!$engine->supportsDatapointsForObject($object)) {
106
96
continue;
107
97
}
108
-
$facts[] = $engine->computeAggregateFacts();
109
-
}
110
-
return array_mergev($facts);
111
-
}
112
-
113
-
private function computeRawFacts(PhabricatorLiskDAO $object) {
114
-
$facts = array();
115
-
foreach ($this->engines as $engine) {
116
-
if (!$engine->shouldComputeRawFactsForObject($object)) {
117
-
continue;
118
-
}
119
-
$facts[] = $engine->computeRawFactsForObject($object);
98
+
$facts[] = $engine->newDatapointsForObject($object);
120
99
}
121
100
122
101
return array_mergev($facts);
123
102
}
124
103
125
-
private function updateRawFacts(array $map) {
104
+
private function updateDatapoints(array $map) {
126
105
foreach ($map as $phid => $facts) {
127
-
assert_instances_of($facts, 'PhabricatorFactRaw');
106
+
assert_instances_of($facts, 'PhabricatorFactIntDatapoint');
128
107
}
129
108
130
109
$phids = array_keys($map);
···
132
111
return;
133
112
}
134
113
135
-
$table = new PhabricatorFactRaw();
114
+
115
+
$fact_keys = array();
116
+
$objects = array();
117
+
foreach ($map as $phid => $facts) {
118
+
foreach ($facts as $fact) {
119
+
$fact_keys[$fact->getKey()] = true;
120
+
121
+
$object_phid = $fact->getObjectPHID();
122
+
$objects[$object_phid] = $object_phid;
123
+
124
+
$dimension_phid = $fact->getDimensionPHID();
125
+
if ($dimension_phid !== null) {
126
+
$objects[$dimension_phid] = $dimension_phid;
127
+
}
128
+
}
129
+
}
130
+
131
+
$key_map = id(new PhabricatorFactKeyDimension())
132
+
->newDimensionMap(array_keys($fact_keys));
133
+
$object_map = id(new PhabricatorFactObjectDimension())
134
+
->newDimensionMap(array_keys($objects));
135
+
136
+
$table = new PhabricatorFactIntDatapoint();
136
137
$conn = $table->establishConnection('w');
137
138
$table_name = $table->getTableName();
138
139
139
140
$sql = array();
140
141
foreach ($map as $phid => $facts) {
141
142
foreach ($facts as $fact) {
143
+
$key_id = $key_map[$fact->getKey()];
144
+
$object_id = $object_map[$fact->getObjectPHID()];
145
+
146
+
$dimension_phid = $fact->getDimensionPHID();
147
+
if ($dimension_phid !== null) {
148
+
$dimension_id = $object_map[$dimension_phid];
149
+
} else {
150
+
$dimension_id = null;
151
+
}
152
+
142
153
$sql[] = qsprintf(
143
154
$conn,
144
-
'(%s, %s, %s, %d, %d, %d)',
145
-
$fact->getFactType(),
146
-
$fact->getObjectPHID(),
147
-
$fact->getObjectA(),
148
-
$fact->getValueX(),
149
-
$fact->getValueY(),
155
+
'(%d, %d, %nd, %d, %d)',
156
+
$key_id,
157
+
$object_id,
158
+
$dimension_id,
159
+
$fact->getValue(),
150
160
$fact->getEpoch());
151
161
}
152
162
}
153
163
164
+
$rebuilt_ids = array_select_keys($object_map, $phids);
165
+
154
166
$table->openTransaction();
155
167
156
168
queryfx(
157
169
$conn,
158
-
'DELETE FROM %T WHERE objectPHID IN (%Ls)',
170
+
'DELETE FROM %T WHERE objectID IN (%Ld)',
159
171
$table_name,
160
-
$phids);
172
+
$rebuilt_ids);
161
173
162
174
if ($sql) {
163
-
foreach (array_chunk($sql, 256) as $chunk) {
175
+
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
164
176
queryfx(
165
177
$conn,
166
178
'INSERT INTO %T
167
-
(factType, objectPHID, objectA, valueX, valueY, epoch)
179
+
(keyID, objectID, dimensionID, value, epoch)
168
180
VALUES %Q',
169
181
$table_name,
170
-
implode(', ', $chunk));
182
+
$chunk);
171
183
}
172
184
}
173
185
174
186
$table->saveTransaction();
175
-
}
176
-
177
-
private function updateAggregateFacts(array $facts) {
178
-
if (!$facts) {
179
-
return;
180
-
}
181
-
182
-
$table = new PhabricatorFactAggregate();
183
-
$conn = $table->establishConnection('w');
184
-
$table_name = $table->getTableName();
185
-
186
-
$sql = array();
187
-
foreach ($facts as $fact) {
188
-
$sql[] = qsprintf(
189
-
$conn,
190
-
'(%s, %s, %d)',
191
-
$fact->getFactType(),
192
-
$fact->getObjectPHID(),
193
-
$fact->getValueX());
194
-
}
195
-
196
-
foreach (array_chunk($sql, 256) as $chunk) {
197
-
queryfx(
198
-
$conn,
199
-
'INSERT INTO %T (factType, objectPHID, valueX) VALUES %Q
200
-
ON DUPLICATE KEY UPDATE valueX = VALUES(valueX)',
201
-
$table_name,
202
-
implode(', ', $chunk));
203
-
}
204
-
205
187
}
206
188
207
189
}
-86
src/applications/fact/engine/PhabricatorFactCountEngine.php
-86
src/applications/fact/engine/PhabricatorFactCountEngine.php
···
1
-
<?php
2
-
3
-
/**
4
-
* Simple fact engine which counts objects.
5
-
*/
6
-
final class PhabricatorFactCountEngine extends PhabricatorFactEngine {
7
-
8
-
public function getFactSpecs(array $fact_types) {
9
-
$results = array();
10
-
foreach ($fact_types as $type) {
11
-
if (!strncmp($type, '+N:', 3)) {
12
-
if ($type == '+N:*') {
13
-
$name = pht('Total Objects');
14
-
} else {
15
-
$name = pht('Total Objects of type %s', substr($type, 3));
16
-
}
17
-
18
-
$results[] = id(new PhabricatorFactSimpleSpec($type))
19
-
->setName($name)
20
-
->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT);
21
-
}
22
-
23
-
if (!strncmp($type, 'N:', 2)) {
24
-
if ($type == 'N:*') {
25
-
$name = pht('Objects');
26
-
} else {
27
-
$name = pht('Objects of type %s', substr($type, 2));
28
-
}
29
-
$results[] = id(new PhabricatorFactSimpleSpec($type))
30
-
->setName($name)
31
-
->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT);
32
-
}
33
-
34
-
}
35
-
return $results;
36
-
}
37
-
38
-
public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) {
39
-
return true;
40
-
}
41
-
42
-
public function computeRawFactsForObject(PhabricatorLiskDAO $object) {
43
-
$facts = array();
44
-
45
-
$phid = $object->getPHID();
46
-
$type = phid_get_type($phid);
47
-
48
-
foreach (array('N:*', 'N:'.$type) as $fact_type) {
49
-
$facts[] = id(new PhabricatorFactRaw())
50
-
->setFactType($fact_type)
51
-
->setObjectPHID($phid)
52
-
->setValueX(1)
53
-
->setEpoch($object->getDateCreated());
54
-
}
55
-
56
-
return $facts;
57
-
}
58
-
59
-
public function shouldComputeAggregateFacts() {
60
-
return true;
61
-
}
62
-
63
-
public function computeAggregateFacts() {
64
-
$table = new PhabricatorFactRaw();
65
-
$table_name = $table->getTableName();
66
-
$conn = $table->establishConnection('r');
67
-
68
-
$counts = queryfx_all(
69
-
$conn,
70
-
'SELECT factType, SUM(valueX) N FROM %T WHERE factType LIKE %>
71
-
GROUP BY factType',
72
-
$table_name,
73
-
'N:');
74
-
75
-
$facts = array();
76
-
foreach ($counts as $count) {
77
-
$facts[] = id(new PhabricatorFactAggregate())
78
-
->setFactType('+'.$count['factType'])
79
-
->setValueX($count['N']);
80
-
}
81
-
82
-
return $facts;
83
-
}
84
-
85
-
86
-
}
+21
-14
src/applications/fact/engine/PhabricatorFactEngine.php
+21
-14
src/applications/fact/engine/PhabricatorFactEngine.php
···
2
2
3
3
abstract class PhabricatorFactEngine extends Phobject {
4
4
5
+
private $factMap;
6
+
5
7
final public static function loadAllEngines() {
6
8
return id(new PhutilClassMapQuery())
7
9
->setAncestorClass(__CLASS__)
8
10
->execute();
9
11
}
10
12
11
-
public function getFactSpecs(array $fact_types) {
12
-
return array();
13
-
}
13
+
abstract public function newFacts();
14
14
15
-
public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) {
16
-
return false;
17
-
}
15
+
abstract public function supportsDatapointsForObject(
16
+
PhabricatorLiskDAO $object);
18
17
19
-
public function computeRawFactsForObject(PhabricatorLiskDAO $object) {
20
-
return array();
21
-
}
18
+
abstract public function newDatapointsForObject(PhabricatorLiskDAO $object);
19
+
20
+
final protected function getFact($key) {
21
+
if ($this->factMap === null) {
22
+
$facts = $this->newFacts();
23
+
$facts = mpull($facts, null, 'getKey');
24
+
$this->factMap = $facts;
25
+
}
22
26
23
-
public function shouldComputeAggregateFacts() {
24
-
return false;
25
-
}
27
+
if (!isset($this->factMap[$key])) {
28
+
throw new Exception(
29
+
pht(
30
+
'Unknown fact ("%s") for engine "%s".',
31
+
$key,
32
+
get_class($this)));
33
+
}
26
34
27
-
public function computeAggregateFacts() {
28
-
return array();
35
+
return $this->factMap[$key];
29
36
}
30
37
31
38
}
-34
src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php
-34
src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php
···
1
-
<?php
2
-
3
-
/**
4
-
* Engine that records the time facts were last updated.
5
-
*/
6
-
final class PhabricatorFactLastUpdatedEngine extends PhabricatorFactEngine {
7
-
8
-
public function getFactSpecs(array $fact_types) {
9
-
$results = array();
10
-
foreach ($fact_types as $type) {
11
-
if ($type == 'updated') {
12
-
$results[] = id(new PhabricatorFactSimpleSpec($type))
13
-
->setName(pht('Facts Last Updated'))
14
-
->setUnit(PhabricatorFactSimpleSpec::UNIT_EPOCH);
15
-
}
16
-
}
17
-
return $results;
18
-
}
19
-
20
-
public function shouldComputeAggregateFacts() {
21
-
return true;
22
-
}
23
-
24
-
public function computeAggregateFacts() {
25
-
$facts = array();
26
-
27
-
$facts[] = id(new PhabricatorFactAggregate())
28
-
->setFactType('updated')
29
-
->setValueX(time());
30
-
31
-
return $facts;
32
-
}
33
-
34
-
}
+34
src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php
+34
src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFactManiphestTaskEngine
4
+
extends PhabricatorFactEngine {
5
+
6
+
public function newFacts() {
7
+
return array(
8
+
id(new PhabricatorPointsFact())
9
+
->setKey('tasks.count.open'),
10
+
);
11
+
}
12
+
13
+
public function supportsDatapointsForObject(PhabricatorLiskDAO $object) {
14
+
return ($object instanceof ManiphestTask);
15
+
}
16
+
17
+
public function newDatapointsForObject(PhabricatorLiskDAO $object) {
18
+
$datapoints = array();
19
+
20
+
$phid = $object->getPHID();
21
+
$type = phid_get_type($phid);
22
+
23
+
$datapoint = $this->getFact('tasks.count.open')
24
+
->newDatapoint();
25
+
26
+
$datapoints[] = $datapoint
27
+
->setObjectPHID($phid)
28
+
->setValue(1)
29
+
->setEpoch($object->getDateCreated());
30
+
31
+
return $datapoints;
32
+
}
33
+
34
+
}
+40
src/applications/fact/fact/PhabricatorFact.php
+40
src/applications/fact/fact/PhabricatorFact.php
···
1
+
<?php
2
+
3
+
abstract class PhabricatorFact extends Phobject {
4
+
5
+
private $key;
6
+
7
+
public static function getAllFacts() {
8
+
$engines = PhabricatorFactEngine::loadAllEngines();
9
+
10
+
$map = array();
11
+
foreach ($engines as $engine) {
12
+
$facts = $engine->newFacts();
13
+
$facts = mpull($facts, null, 'getKey');
14
+
$map += $facts;
15
+
}
16
+
17
+
return $map;
18
+
}
19
+
20
+
final public function setKey($key) {
21
+
$this->key = $key;
22
+
return $this;
23
+
}
24
+
25
+
final public function getKey() {
26
+
return $this->key;
27
+
}
28
+
29
+
final public function getName() {
30
+
return pht('Fact "%s"', $this->getKey());
31
+
}
32
+
33
+
final public function newDatapoint() {
34
+
return $this->newTemplateDatapoint()
35
+
->setKey($this->getKey());
36
+
}
37
+
38
+
abstract protected function newTemplateDatapoint();
39
+
40
+
}
+9
src/applications/fact/fact/PhabricatorPointsFact.php
+9
src/applications/fact/fact/PhabricatorPointsFact.php
-4
src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php
-4
src/applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php
+7
-2
src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php
+7
-2
src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php
···
23
23
}
24
24
25
25
$tables = array();
26
-
$tables[] = new PhabricatorFactRaw();
27
-
$tables[] = new PhabricatorFactAggregate();
26
+
$tables[] = new PhabricatorFactCursor();
27
+
28
+
$tables[] = new PhabricatorFactIntDatapoint();
29
+
30
+
$tables[] = new PhabricatorFactObjectDimension();
31
+
$tables[] = new PhabricatorFactKeyDimension();
32
+
28
33
foreach ($tables as $table) {
29
34
$conn = $table->establishConnection('w');
30
35
$name = $table->getTableName();
-47
src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php
-47
src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php
···
1
-
<?php
2
-
3
-
final class PhabricatorFactManagementStatusWorkflow
4
-
extends PhabricatorFactManagementWorkflow {
5
-
6
-
protected function didConstruct() {
7
-
$this
8
-
->setName('status')
9
-
->setSynopsis(pht('Show status of fact data.'))
10
-
->setArguments(array());
11
-
}
12
-
13
-
public function execute(PhutilArgumentParser $args) {
14
-
$console = PhutilConsole::getConsole();
15
-
16
-
$map = array(
17
-
'raw' => new PhabricatorFactRaw(),
18
-
'agg' => new PhabricatorFactAggregate(),
19
-
);
20
-
21
-
foreach ($map as $type => $table) {
22
-
$conn = $table->establishConnection('r');
23
-
$name = $table->getTableName();
24
-
25
-
$row = queryfx_one(
26
-
$conn,
27
-
'SELECT COUNT(*) N FROM %T',
28
-
$name);
29
-
30
-
$n = $row['N'];
31
-
32
-
switch ($type) {
33
-
case 'raw':
34
-
$desc = pht('There are %d raw fact(s) in storage.', $n);
35
-
break;
36
-
case 'agg':
37
-
$desc = pht('There are %d aggregate fact(s) in storage.', $n);
38
-
break;
39
-
}
40
-
41
-
$console->writeOut("%s\n", $desc);
42
-
}
43
-
44
-
return 0;
45
-
}
46
-
47
-
}
-38
src/applications/fact/spec/PhabricatorFactSimpleSpec.php
-38
src/applications/fact/spec/PhabricatorFactSimpleSpec.php
···
1
-
<?php
2
-
3
-
final class PhabricatorFactSimpleSpec extends PhabricatorFactSpec {
4
-
5
-
private $type;
6
-
private $name;
7
-
private $unit;
8
-
9
-
public function __construct($type) {
10
-
$this->type = $type;
11
-
}
12
-
13
-
public function getType() {
14
-
return $this->type;
15
-
}
16
-
17
-
public function setUnit($unit) {
18
-
$this->unit = $unit;
19
-
return $this;
20
-
}
21
-
22
-
public function getUnit() {
23
-
return $this->unit;
24
-
}
25
-
26
-
public function setName($name) {
27
-
$this->name = $name;
28
-
return $this;
29
-
}
30
-
31
-
public function getName() {
32
-
if ($this->name !== null) {
33
-
return $this->name;
34
-
}
35
-
return parent::getName();
36
-
}
37
-
38
-
}
-53
src/applications/fact/spec/PhabricatorFactSpec.php
-53
src/applications/fact/spec/PhabricatorFactSpec.php
···
1
-
<?php
2
-
3
-
abstract class PhabricatorFactSpec extends Phobject {
4
-
5
-
const UNIT_COUNT = 'unit-count';
6
-
const UNIT_EPOCH = 'unit-epoch';
7
-
8
-
public static function newSpecsForFactTypes(
9
-
array $engines,
10
-
array $fact_types) {
11
-
assert_instances_of($engines, 'PhabricatorFactEngine');
12
-
13
-
$map = array();
14
-
foreach ($engines as $engine) {
15
-
$specs = $engine->getFactSpecs($fact_types);
16
-
$specs = mpull($specs, null, 'getType');
17
-
$map += $specs;
18
-
}
19
-
20
-
foreach ($fact_types as $type) {
21
-
if (empty($map[$type])) {
22
-
$map[$type] = new PhabricatorFactSimpleSpec($type);
23
-
}
24
-
}
25
-
26
-
return $map;
27
-
}
28
-
29
-
abstract public function getType();
30
-
31
-
public function getUnit() {
32
-
return null;
33
-
}
34
-
35
-
public function getName() {
36
-
return pht(
37
-
'Fact (%s)',
38
-
$this->getType());
39
-
}
40
-
41
-
public function formatValueForDisplay(PhabricatorUser $user, $value) {
42
-
$unit = $this->getUnit();
43
-
switch ($unit) {
44
-
case self::UNIT_COUNT:
45
-
return number_format($value);
46
-
case self::UNIT_EPOCH:
47
-
return phabricator_datetime($value, $user);
48
-
default:
49
-
return $value;
50
-
}
51
-
}
52
-
53
-
}
+85
src/applications/fact/storage/PhabricatorFactDimension.php
+85
src/applications/fact/storage/PhabricatorFactDimension.php
···
1
+
<?php
2
+
3
+
abstract class PhabricatorFactDimension extends PhabricatorFactDAO {
4
+
5
+
abstract protected function getDimensionColumnName();
6
+
7
+
final public function newDimensionID($key) {
8
+
$map = $this->newDimensionMap(array($key));
9
+
return $map[$key];
10
+
}
11
+
12
+
final public function newDimensionMap(array $keys) {
13
+
if (!$keys) {
14
+
return array();
15
+
}
16
+
17
+
$conn = $this->establishConnection('r');
18
+
$column = $this->getDimensionColumnName();
19
+
20
+
$rows = queryfx_all(
21
+
$conn,
22
+
'SELECT id, %C FROM %T WHERE %C IN (%Ls)',
23
+
$column,
24
+
$this->getTableName(),
25
+
$column,
26
+
$keys);
27
+
$rows = ipull($rows, 'id', $column);
28
+
29
+
$map = array();
30
+
$need = array();
31
+
foreach ($keys as $key) {
32
+
if (isset($rows[$key])) {
33
+
$map[$key] = (int)$rows[$key];
34
+
} else {
35
+
$need[] = $key;
36
+
}
37
+
}
38
+
39
+
if (!$need) {
40
+
return $map;
41
+
}
42
+
43
+
$sql = array();
44
+
foreach ($need as $key) {
45
+
$sql[] = qsprintf(
46
+
$conn,
47
+
'(%s)',
48
+
$key);
49
+
}
50
+
51
+
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
52
+
queryfx(
53
+
$conn,
54
+
'INSERT IGNORE INTO %T (%C) VALUES %Q',
55
+
$this->getTableName(),
56
+
$column,
57
+
$chunk);
58
+
}
59
+
60
+
$rows = queryfx_all(
61
+
$conn,
62
+
'SELECT id, %C FROM %T WHERE %C IN (%Ls)',
63
+
$column,
64
+
$this->getTableName(),
65
+
$column,
66
+
$need);
67
+
$rows = ipull($rows, 'id', $column);
68
+
69
+
foreach ($keys as $key) {
70
+
if (isset($rows[$key])) {
71
+
$map[$key] = (int)$rows[$key];
72
+
} else {
73
+
throw new Exception(
74
+
pht(
75
+
'Failed to load or generate dimension ID ("%s") for dimension '.
76
+
'key "%s".',
77
+
get_class($this),
78
+
$key));
79
+
}
80
+
}
81
+
82
+
return $map;
83
+
}
84
+
85
+
}
+61
src/applications/fact/storage/PhabricatorFactIntDatapoint.php
+61
src/applications/fact/storage/PhabricatorFactIntDatapoint.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFactIntDatapoint extends PhabricatorFactDAO {
4
+
5
+
protected $keyID;
6
+
protected $objectID;
7
+
protected $dimensionID;
8
+
protected $value;
9
+
protected $epoch;
10
+
11
+
private $key;
12
+
private $objectPHID;
13
+
private $dimensionPHID;
14
+
15
+
protected function getConfiguration() {
16
+
return array(
17
+
self::CONFIG_TIMESTAMPS => false,
18
+
self::CONFIG_COLUMN_SCHEMA => array(
19
+
'id' => 'auto64',
20
+
'dimensionID' => 'id?',
21
+
'value' => 'sint64',
22
+
),
23
+
self::CONFIG_KEY_SCHEMA => array(
24
+
'key_dimension' => array(
25
+
'columns' => array('keyID', 'dimensionID'),
26
+
),
27
+
'key_object' => array(
28
+
'columns' => array('objectID'),
29
+
),
30
+
),
31
+
) + parent::getConfiguration();
32
+
}
33
+
34
+
public function setKey($key) {
35
+
$this->key = $key;
36
+
return $this;
37
+
}
38
+
39
+
public function getKey() {
40
+
return $this->key;
41
+
}
42
+
43
+
public function setObjectPHID($object_phid) {
44
+
$this->objectPHID = $object_phid;
45
+
return $this;
46
+
}
47
+
48
+
public function getObjectPHID() {
49
+
return $this->objectPHID;
50
+
}
51
+
52
+
public function setDimensionPHID($dimension_phid) {
53
+
$this->dimensionPHID = $dimension_phid;
54
+
return $this;
55
+
}
56
+
57
+
public function getDimensionPHID() {
58
+
return $this->dimensionPHID;
59
+
}
60
+
61
+
}
+27
src/applications/fact/storage/PhabricatorFactKeyDimension.php
+27
src/applications/fact/storage/PhabricatorFactKeyDimension.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFactKeyDimension
4
+
extends PhabricatorFactDimension {
5
+
6
+
protected $factKey;
7
+
8
+
protected function getConfiguration() {
9
+
return array(
10
+
self::CONFIG_TIMESTAMPS => false,
11
+
self::CONFIG_COLUMN_SCHEMA => array(
12
+
'factKey' => 'text64',
13
+
),
14
+
self::CONFIG_KEY_SCHEMA => array(
15
+
'key_factkey' => array(
16
+
'columns' => array('factKey'),
17
+
'unique' => true,
18
+
),
19
+
),
20
+
) + parent::getConfiguration();
21
+
}
22
+
23
+
protected function getDimensionColumnName() {
24
+
return 'factKey';
25
+
}
26
+
27
+
}
+25
src/applications/fact/storage/PhabricatorFactObjectDimension.php
+25
src/applications/fact/storage/PhabricatorFactObjectDimension.php
···
1
+
<?php
2
+
3
+
final class PhabricatorFactObjectDimension
4
+
extends PhabricatorFactDimension {
5
+
6
+
protected $objectPHID;
7
+
8
+
protected function getConfiguration() {
9
+
return array(
10
+
self::CONFIG_TIMESTAMPS => false,
11
+
self::CONFIG_COLUMN_SCHEMA => array(),
12
+
self::CONFIG_KEY_SCHEMA => array(
13
+
'key_object' => array(
14
+
'columns' => array('objectPHID'),
15
+
'unique' => true,
16
+
),
17
+
),
18
+
) + parent::getConfiguration();
19
+
}
20
+
21
+
protected function getDimensionColumnName() {
22
+
return 'objectPHID';
23
+
}
24
+
25
+
}
-6
src/applications/ponder/application/PhabricatorPonderApplication.php
-6
src/applications/ponder/application/PhabricatorPonderApplication.php