@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a "Database Cluster Status" console in Config
Summary: Ref T4571. The configuration option still doesn't do anything, but add a status panel for basic setup monitoring.
Test Plan:
Here's what a good version looks like:
{F1212291}
Also faked most of the errors it can detect and got helpful diagnostic messages like this:
{F1212292}
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T4571
Differential Revision: https://secure.phabricator.com/D15667
+518
-1
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
2029
2029
'PhabricatorConfigAllController' => 'applications/config/controller/PhabricatorConfigAllController.php',
2030
2030
'PhabricatorConfigApplication' => 'applications/config/application/PhabricatorConfigApplication.php',
2031
2031
'PhabricatorConfigCacheController' => 'applications/config/controller/PhabricatorConfigCacheController.php',
2032
+
'PhabricatorConfigClusterDatabasesController' => 'applications/config/controller/PhabricatorConfigClusterDatabasesController.php',
2032
2033
'PhabricatorConfigCollectorsModule' => 'applications/config/module/PhabricatorConfigCollectorsModule.php',
2033
2034
'PhabricatorConfigColumnSchema' => 'applications/config/schema/PhabricatorConfigColumnSchema.php',
2034
2035
'PhabricatorConfigConfigPHIDType' => 'applications/config/phid/PhabricatorConfigConfigPHIDType.php',
···
2235
2236
'PhabricatorDashboardViewController' => 'applications/dashboard/controller/PhabricatorDashboardViewController.php',
2236
2237
'PhabricatorDataCacheSpec' => 'applications/cache/spec/PhabricatorDataCacheSpec.php',
2237
2238
'PhabricatorDataNotAttachedException' => 'infrastructure/storage/lisk/PhabricatorDataNotAttachedException.php',
2239
+
'PhabricatorDatabaseRef' => 'infrastructure/cluster/PhabricatorDatabaseRef.php',
2238
2240
'PhabricatorDatabaseSetupCheck' => 'applications/config/check/PhabricatorDatabaseSetupCheck.php',
2239
2241
'PhabricatorDatasourceEditField' => 'applications/transactions/editfield/PhabricatorDatasourceEditField.php',
2240
2242
'PhabricatorDatasourceEditType' => 'applications/transactions/edittype/PhabricatorDatasourceEditType.php',
···
6441
6443
'PhabricatorConfigAllController' => 'PhabricatorConfigController',
6442
6444
'PhabricatorConfigApplication' => 'PhabricatorApplication',
6443
6445
'PhabricatorConfigCacheController' => 'PhabricatorConfigController',
6446
+
'PhabricatorConfigClusterDatabasesController' => 'PhabricatorConfigController',
6444
6447
'PhabricatorConfigCollectorsModule' => 'PhabricatorConfigModule',
6445
6448
'PhabricatorConfigColumnSchema' => 'PhabricatorConfigStorageSchema',
6446
6449
'PhabricatorConfigConfigPHIDType' => 'PhabricatorPHIDType',
···
6683
6686
'PhabricatorDashboardViewController' => 'PhabricatorDashboardController',
6684
6687
'PhabricatorDataCacheSpec' => 'PhabricatorCacheSpec',
6685
6688
'PhabricatorDataNotAttachedException' => 'Exception',
6689
+
'PhabricatorDatabaseRef' => 'Phobject',
6686
6690
'PhabricatorDatabaseSetupCheck' => 'PhabricatorSetupCheck',
6687
6691
'PhabricatorDatasourceEditField' => 'PhabricatorTokenizerEditField',
6688
6692
'PhabricatorDatasourceEditType' => 'PhabricatorPHIDListEditType',
+3
src/applications/config/application/PhabricatorConfigApplication.php
+3
src/applications/config/application/PhabricatorConfigApplication.php
+183
src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php
+183
src/applications/config/controller/PhabricatorConfigClusterDatabasesController.php
···
1
+
<?php
2
+
3
+
final class PhabricatorConfigClusterDatabasesController
4
+
extends PhabricatorConfigController {
5
+
6
+
public function handleRequest(AphrontRequest $request) {
7
+
$nav = $this->buildSideNavView();
8
+
$nav->selectFilter('cluster/databases/');
9
+
10
+
$title = pht('Cluster Databases');
11
+
12
+
$crumbs = $this
13
+
->buildApplicationCrumbs($nav)
14
+
->addTextCrumb(pht('Cluster Databases'));
15
+
16
+
$database_status = $this->buildClusterDatabaseStatus();
17
+
18
+
$view = id(new PHUITwoColumnView())
19
+
->setNavigation($nav)
20
+
->setMainColumn($database_status);
21
+
22
+
return $this->newPage()
23
+
->setTitle($title)
24
+
->setCrumbs($crumbs)
25
+
->appendChild($view);
26
+
}
27
+
28
+
private function buildClusterDatabaseStatus() {
29
+
$viewer = $this->getViewer();
30
+
31
+
$databases = PhabricatorDatabaseRef::queryAll();
32
+
$connection_map = PhabricatorDatabaseRef::getConnectionStatusMap();
33
+
$replica_map = PhabricatorDatabaseRef::getReplicaStatusMap();
34
+
Javelin::initBehavior('phabricator-tooltips');
35
+
36
+
$rows = array();
37
+
foreach ($databases as $database) {
38
+
if ($database->getIsMaster()) {
39
+
$role_icon = id(new PHUIIconView())
40
+
->setIcon('fa-database sky')
41
+
->addSigil('has-tooltip')
42
+
->setMetadata(
43
+
array(
44
+
'tip' => pht('Master'),
45
+
));
46
+
} else {
47
+
$role_icon = id(new PHUIIconView())
48
+
->setIcon('fa-download')
49
+
->addSigil('has-tooltip')
50
+
->setMetadata(
51
+
array(
52
+
'tip' => pht('Replica'),
53
+
));
54
+
}
55
+
56
+
if ($database->getDisabled()) {
57
+
$conn_icon = 'fa-times';
58
+
$conn_color = 'grey';
59
+
$conn_label = pht('Disabled');
60
+
} else {
61
+
$status = $database->getConnectionStatus();
62
+
63
+
$info = idx($connection_map, $status, array());
64
+
$conn_icon = idx($info, 'icon');
65
+
$conn_color = idx($info, 'color');
66
+
$conn_label = idx($info, 'label');
67
+
68
+
if ($status === PhabricatorDatabaseRef::STATUS_OKAY) {
69
+
$latency = $database->getConnectionLatency();
70
+
$latency = (int)(1000000 * $latency);
71
+
$conn_label = pht('%s us', new PhutilNumber($latency));
72
+
}
73
+
}
74
+
75
+
$connection = array(
76
+
id(new PHUIIconView())->setIcon("{$conn_icon} {$conn_color}"),
77
+
' ',
78
+
$conn_label,
79
+
);
80
+
81
+
if ($database->getDisabled()) {
82
+
$replica_icon = 'fa-times';
83
+
$replica_color = 'grey';
84
+
$replica_label = pht('Disabled');
85
+
} else {
86
+
$status = $database->getReplicaStatus();
87
+
88
+
$info = idx($replica_map, $status, array());
89
+
$replica_icon = idx($info, 'icon');
90
+
$replica_color = idx($info, 'color');
91
+
$replica_label = idx($info, 'label');
92
+
93
+
if ($database->getIsMaster()) {
94
+
if ($status === PhabricatorDatabaseRef::REPLICATION_OKAY) {
95
+
$replica_icon = 'fa-database';
96
+
}
97
+
} else {
98
+
switch ($status) {
99
+
case PhabricatorDatabaseRef::REPLICATION_OKAY:
100
+
case PhabricatorDatabaseRef::REPLICATION_SLOW:
101
+
$delay = $database->getReplicaDelay();
102
+
if ($delay) {
103
+
$replica_label = pht('%ss Behind', new PhutilNumber($delay));
104
+
} else {
105
+
$replica_label = pht('Up to Date');
106
+
}
107
+
break;
108
+
}
109
+
}
110
+
}
111
+
112
+
$replication = array(
113
+
id(new PHUIIconView())->setIcon("{$replica_icon} {$replica_color}"),
114
+
' ',
115
+
$replica_label,
116
+
);
117
+
118
+
$messages = array();
119
+
120
+
$conn_message = $database->getConnectionMessage();
121
+
if ($conn_message) {
122
+
$messages[] = $conn_message;
123
+
}
124
+
125
+
$replica_message = $database->getReplicaMessage();
126
+
if ($replica_message) {
127
+
$messages[] = $replica_message;
128
+
}
129
+
130
+
$messages = phutil_implode_html(phutil_tag('br'), $messages);
131
+
132
+
$rows[] = array(
133
+
$role_icon,
134
+
$database->getHost(),
135
+
$database->getPort(),
136
+
$database->getUser(),
137
+
$connection,
138
+
$replication,
139
+
$messages,
140
+
);
141
+
}
142
+
143
+
$table = id(new AphrontTableView($rows))
144
+
->setNoDataString(
145
+
pht('Phabricator is not configured in cluster mode.'))
146
+
->setHeaders(
147
+
array(
148
+
null,
149
+
pht('Host'),
150
+
pht('Port'),
151
+
pht('User'),
152
+
pht('Connection'),
153
+
pht('Replication'),
154
+
pht('Messages'),
155
+
))
156
+
->setColumnClasses(
157
+
array(
158
+
null,
159
+
null,
160
+
null,
161
+
null,
162
+
null,
163
+
null,
164
+
'wide',
165
+
));
166
+
167
+
$doc_href = PhabricatorEnv::getDoclink('Cluster: Databases');
168
+
169
+
$header = id(new PHUIHeaderView())
170
+
->setHeader(pht('Cluster Database Status'))
171
+
->addActionLink(
172
+
id(new PHUIButtonView())
173
+
->setIcon('fa-book')
174
+
->setHref($doc_href)
175
+
->setTag('a')
176
+
->setText(pht('Database Clustering Documentation')));
177
+
178
+
return id(new PHUIObjectBoxView())
179
+
->setHeader($header)
180
+
->setTable($table);
181
+
}
182
+
183
+
}
+2
src/applications/config/controller/PhabricatorConfigController.php
+2
src/applications/config/controller/PhabricatorConfigController.php
···
22
22
$nav->addFilter('dbissue/', pht('Database Issues'));
23
23
$nav->addLabel(pht('Cache'));
24
24
$nav->addFilter('cache/', pht('Cache Status'));
25
+
$nav->addLabel(pht('Cluster'));
26
+
$nav->addFilter('cluster/databases/', pht('Cluster Databases'));
25
27
$nav->addLabel(pht('Welcome'));
26
28
$nav->addFilter('welcome/', pht('Welcome Screen'));
27
29
$nav->addLabel(pht('Modules'));
+5
-1
src/docs/user/cluster/cluster_databases.diviner
+5
-1
src/docs/user/cluster/cluster_databases.diviner
···
65
65
Monitoring and Testing
66
66
======================
67
67
68
-
TODO: Write this part.
68
+
You can monitor replicas in {nav Config > Cluster Databases}. This interface
69
+
shows you a quick overview of replicas and their health, and can detect some
70
+
common issues with replication.
71
+
72
+
TODO: Write more stuff here.
69
73
70
74
Degradation to Read-Only Mode
71
75
=============================
+321
src/infrastructure/cluster/PhabricatorDatabaseRef.php
+321
src/infrastructure/cluster/PhabricatorDatabaseRef.php
···
1
+
<?php
2
+
3
+
final class PhabricatorDatabaseRef
4
+
extends Phobject {
5
+
6
+
const STATUS_OKAY = 'okay';
7
+
const STATUS_FAIL = 'fail';
8
+
const STATUS_AUTH = 'auth';
9
+
const STATUS_REPLICATION_CLIENT = 'replication-client';
10
+
11
+
const REPLICATION_OKAY = 'okay';
12
+
const REPLICATION_MASTER_REPLICA = 'master-replica';
13
+
const REPLICATION_REPLICA_NONE = 'replica-none';
14
+
const REPLICATION_SLOW = 'replica-slow';
15
+
16
+
private $host;
17
+
private $port;
18
+
private $user;
19
+
private $pass;
20
+
private $disabled;
21
+
private $isMaster;
22
+
23
+
private $connectionLatency;
24
+
private $connectionStatus;
25
+
private $connectionMessage;
26
+
27
+
private $replicaStatus;
28
+
private $replicaMessage;
29
+
private $replicaDelay;
30
+
31
+
public function setHost($host) {
32
+
$this->host = $host;
33
+
return $this;
34
+
}
35
+
36
+
public function getHost() {
37
+
return $this->host;
38
+
}
39
+
40
+
public function setPort($port) {
41
+
$this->port = $port;
42
+
return $this;
43
+
}
44
+
45
+
public function getPort() {
46
+
return $this->port;
47
+
}
48
+
49
+
public function setUser($user) {
50
+
$this->user = $user;
51
+
return $this;
52
+
}
53
+
54
+
public function getUser() {
55
+
return $this->user;
56
+
}
57
+
58
+
public function setPass(PhutilOpaqueEnvelope $pass) {
59
+
$this->pass = $pass;
60
+
return $this;
61
+
}
62
+
63
+
public function getPass() {
64
+
return $this->pass;
65
+
}
66
+
67
+
public function setIsMaster($is_master) {
68
+
$this->isMaster = $is_master;
69
+
return $this;
70
+
}
71
+
72
+
public function getIsMaster() {
73
+
return $this->isMaster;
74
+
}
75
+
76
+
public function setDisabled($disabled) {
77
+
$this->disabled = $disabled;
78
+
return $this;
79
+
}
80
+
81
+
public function getDisabled() {
82
+
return $this->disabled;
83
+
}
84
+
85
+
public function setConnectionLatency($connection_latency) {
86
+
$this->connectionLatency = $connection_latency;
87
+
return $this;
88
+
}
89
+
90
+
public function getConnectionLatency() {
91
+
return $this->connectionLatency;
92
+
}
93
+
94
+
public function setConnectionStatus($connection_status) {
95
+
$this->connectionStatus = $connection_status;
96
+
return $this;
97
+
}
98
+
99
+
public function getConnectionStatus() {
100
+
if ($this->connectionStatus === null) {
101
+
throw new PhutilInvalidStateException('queryAll');
102
+
}
103
+
104
+
return $this->connectionStatus;
105
+
}
106
+
107
+
public function setConnectionMessage($connection_message) {
108
+
$this->connectionMessage = $connection_message;
109
+
return $this;
110
+
}
111
+
112
+
public function getConnectionMessage() {
113
+
return $this->connectionMessage;
114
+
}
115
+
116
+
public function setReplicaStatus($replica_status) {
117
+
$this->replicaStatus = $replica_status;
118
+
return $this;
119
+
}
120
+
121
+
public function getReplicaStatus() {
122
+
return $this->replicaStatus;
123
+
}
124
+
125
+
public function setReplicaMessage($replica_message) {
126
+
$this->replicaMessage = $replica_message;
127
+
return $this;
128
+
}
129
+
130
+
public function getReplicaMessage() {
131
+
return $this->replicaMessage;
132
+
}
133
+
134
+
public function setReplicaDelay($replica_delay) {
135
+
$this->replicaDelay = $replica_delay;
136
+
return $this;
137
+
}
138
+
139
+
public function getReplicaDelay() {
140
+
return $this->replicaDelay;
141
+
}
142
+
143
+
public static function getConnectionStatusMap() {
144
+
return array(
145
+
self::STATUS_OKAY => array(
146
+
'icon' => 'fa-exchange',
147
+
'color' => 'green',
148
+
'label' => pht('Okay'),
149
+
),
150
+
self::STATUS_FAIL => array(
151
+
'icon' => 'fa-times',
152
+
'color' => 'red',
153
+
'label' => pht('Failed'),
154
+
),
155
+
self::STATUS_AUTH => array(
156
+
'icon' => 'fa-key',
157
+
'color' => 'red',
158
+
'label' => pht('Invalid Credentials'),
159
+
),
160
+
self::STATUS_REPLICATION_CLIENT => array(
161
+
'icon' => 'fa-eye-slash',
162
+
'color' => 'yellow',
163
+
'label' => pht('Missing Permission'),
164
+
),
165
+
);
166
+
}
167
+
168
+
public static function getReplicaStatusMap() {
169
+
return array(
170
+
self::REPLICATION_OKAY => array(
171
+
'icon' => 'fa-download',
172
+
'color' => 'green',
173
+
'label' => pht('Okay'),
174
+
),
175
+
self::REPLICATION_MASTER_REPLICA => array(
176
+
'icon' => 'fa-database',
177
+
'color' => 'red',
178
+
'label' => pht('Replicating Master'),
179
+
),
180
+
self::REPLICATION_REPLICA_NONE => array(
181
+
'icon' => 'fa-download',
182
+
'color' => 'red',
183
+
'label' => pht('Not Replicating'),
184
+
),
185
+
self::REPLICATION_SLOW => array(
186
+
'icon' => 'fa-hourglass',
187
+
'color' => 'red',
188
+
'label' => pht('Slow Replication'),
189
+
),
190
+
);
191
+
}
192
+
193
+
public static function loadAll() {
194
+
$refs = array();
195
+
196
+
$default_port = PhabricatorEnv::getEnvConfig('mysql.port');
197
+
$default_port = nonempty($default_port, 3306);
198
+
199
+
$default_user = PhabricatorEnv::getEnvConfig('mysql.user');
200
+
201
+
$default_pass = PhabricatorEnv::getEnvConfig('mysql.pass');
202
+
$default_pass = new PhutilOpaqueEnvelope($default_pass);
203
+
204
+
$config = PhabricatorEnv::getEnvConfig('cluster.databases');
205
+
foreach ($config as $server) {
206
+
$host = $server['host'];
207
+
$port = idx($server, 'port', $default_port);
208
+
$user = idx($server, 'user', $default_user);
209
+
$disabled = idx($server, 'disabled', false);
210
+
211
+
$pass = idx($server, 'pass');
212
+
if ($pass) {
213
+
$pass = new PhutilOpaqueEnvelope($pass);
214
+
} else {
215
+
$pass = clone $default_pass;
216
+
}
217
+
218
+
$role = $server['role'];
219
+
220
+
$ref = id(new self())
221
+
->setHost($host)
222
+
->setPort($port)
223
+
->setUser($user)
224
+
->setPass($pass)
225
+
->setDisabled($disabled)
226
+
->setIsMaster(($role == 'master'));
227
+
228
+
$refs[] = $ref;
229
+
}
230
+
231
+
return $refs;
232
+
}
233
+
234
+
public static function queryAll() {
235
+
$refs = self::loadAll();
236
+
237
+
foreach ($refs as $ref) {
238
+
if ($ref->getDisabled()) {
239
+
continue;
240
+
}
241
+
242
+
$conn = $ref->newConnection();
243
+
244
+
$t_start = microtime(true);
245
+
try {
246
+
$replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS');
247
+
$ref->setConnectionStatus(self::STATUS_OKAY);
248
+
} catch (AphrontAccessDeniedQueryException $ex) {
249
+
$ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT);
250
+
$ref->setConnectionMessage(
251
+
pht(
252
+
'No permission to run "SHOW SLAVE STATUS". Grant this user '.
253
+
'"REPLICATION CLIENT" permission to allow Phabricator to '.
254
+
'monitor replica health.'));
255
+
} catch (AphrontInvalidCredentialsQueryException $ex) {
256
+
$ref->setConnectionStatus(self::STATUS_AUTH);
257
+
$ref->setConnectionMessage($ex->getMessage());
258
+
} catch (AphrontQueryException $ex) {
259
+
$ref->setConnectionStatus(self::STATUS_FAIL);
260
+
261
+
$class = get_class($ex);
262
+
$message = $ex->getMessage();
263
+
$ref->setConnectionMessage(
264
+
pht(
265
+
'%s: %s',
266
+
get_class($ex),
267
+
$ex->getMessage()));
268
+
}
269
+
$t_end = microtime(true);
270
+
$ref->setConnectionLatency($t_end - $t_start);
271
+
272
+
$is_replica = (bool)$replica_status;
273
+
if ($ref->getIsMaster() && $is_replica) {
274
+
$ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA);
275
+
$ref->setReplicaMessage(
276
+
pht(
277
+
'This host has a "master" role, but is replicating data from '.
278
+
'another host ("%s")!',
279
+
idx($replica_status, 'Master_Host')));
280
+
} else if (!$ref->getIsMaster() && !$is_replica) {
281
+
$ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE);
282
+
$ref->setReplicaMessage(
283
+
pht(
284
+
'This host has a "replica" role, but is not replicating data '.
285
+
'from a master (no output from "SHOW SLAVE STATUS").'));
286
+
} else {
287
+
$ref->setReplicaStatus(self::REPLICATION_OKAY);
288
+
}
289
+
290
+
if ($is_replica) {
291
+
$latency = (int)idx($replica_status, 'Seconds_Behind_Master');
292
+
$ref->setReplicaDelay($latency);
293
+
if ($latency > 30) {
294
+
$ref->setReplicaStatus(self::REPLICATION_SLOW);
295
+
$ref->setReplicaMessage(
296
+
pht(
297
+
'This replica is lagging far behind the master. Data is at '.
298
+
'risk!'));
299
+
}
300
+
}
301
+
}
302
+
303
+
return $refs;
304
+
}
305
+
306
+
protected function newConnection() {
307
+
return PhabricatorEnv::newObjectFromConfig(
308
+
'mysql.implementation',
309
+
array(
310
+
array(
311
+
'user' => $this->getUser(),
312
+
'pass' => $this->getPass(),
313
+
'host' => $this->getHost(),
314
+
'port' => $this->getPort(),
315
+
'database' => null,
316
+
'retries' => 0,
317
+
),
318
+
));
319
+
}
320
+
321
+
}