@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Lay `cluster.databases` configuration groundwork for database clustering
Summary:
Ref T4571. This adds a new option which allows you to upgrade your one-host configuration to a multi-host configuration by configuring it.
Doing this currently does nothing. I wrote a lot of words about what it is //supposed// to do in the future, though.
Test Plan:
- Tried to configure the option in all the possible bad ways, got errors.
- Read documentation.
Reviewers: chad
Reviewed By: chad
Subscribers: eadler
Maniphest Tasks: T4571
Differential Revision: https://secure.phabricator.com/D15663
+345
-7
+12
-4
resources/celerity/map.php
+12
-4
resources/celerity/map.php
···
7
7
*/
8
8
return array(
9
9
'names' => array(
10
-
'core.pkg.css' => '82cefddc',
11
-
'core.pkg.js' => 'e5484f37',
10
+
'core.pkg.css' => '35e4a99a',
11
+
'core.pkg.js' => '8a616602',
12
12
'darkconsole.pkg.js' => 'e7393ebb',
13
13
'differential.pkg.css' => '7ba78475',
14
14
'differential.pkg.js' => 'd0cd0df6',
···
22
22
'rsrc/css/aphront/lightbox-attachment.css' => '7acac05d',
23
23
'rsrc/css/aphront/list-filter-view.css' => '5d6f0526',
24
24
'rsrc/css/aphront/multi-column.css' => 'fd18389d',
25
-
'rsrc/css/aphront/notification.css' => '7f684b62',
25
+
'rsrc/css/aphront/notification.css' => '3f6c89c9',
26
26
'rsrc/css/aphront/panel-view.css' => '8427b78d',
27
27
'rsrc/css/aphront/phabricator-nav-view.css' => 'ac79a758',
28
28
'rsrc/css/aphront/table-view.css' => '9258e19f',
···
494
494
'rsrc/js/core/behavior-oncopy.js' => '2926fff2',
495
495
'rsrc/js/core/behavior-phabricator-nav.js' => '56a1ca03',
496
496
'rsrc/js/core/behavior-phabricator-remarkup-assist.js' => '340c8eff',
497
+
'rsrc/js/core/behavior-read-only-warning.js' => 'f8ea359c',
497
498
'rsrc/js/core/behavior-refresh-csrf.js' => 'ab2f381b',
498
499
'rsrc/js/core/behavior-remarkup-preview.js' => '4b700e9e',
499
500
'rsrc/js/core/behavior-reorder-applications.js' => '76b9fc3e',
···
666
667
'javelin-behavior-project-boards' => '14a1faae',
667
668
'javelin-behavior-project-create' => '065227cc',
668
669
'javelin-behavior-quicksand-blacklist' => '7927a7d3',
670
+
'javelin-behavior-read-only-warning' => 'f8ea359c',
669
671
'javelin-behavior-recurring-edit' => '5f1c4d5f',
670
672
'javelin-behavior-refresh-csrf' => 'ab2f381b',
671
673
'javelin-behavior-releeph-preview-branch' => 'b2b4fbaf',
···
766
768
'phabricator-main-menu-view' => 'd00a795a',
767
769
'phabricator-nav-view-css' => 'ac79a758',
768
770
'phabricator-notification' => 'ccf1cbf8',
769
-
'phabricator-notification-css' => '7f684b62',
771
+
'phabricator-notification-css' => '3f6c89c9',
770
772
'phabricator-notification-menu-css' => 'f31c0bde',
771
773
'phabricator-object-selector-css' => '85ee8ce6',
772
774
'phabricator-phtize' => 'd254d646',
···
2109
2111
'javelin-util',
2110
2112
'phabricator-busy',
2111
2113
),
2114
+
'f8ea359c' => array(
2115
+
'javelin-behavior',
2116
+
'javelin-uri',
2117
+
'phabricator-notification',
2118
+
),
2112
2119
'fa0f4fc2' => array(
2113
2120
'javelin-behavior',
2114
2121
'javelin-dom',
···
2284
2291
'javelin-quicksand',
2285
2292
'javelin-behavior-quicksand-blacklist',
2286
2293
'javelin-behavior-high-security-warning',
2294
+
'javelin-behavior-read-only-warning',
2287
2295
'javelin-scrollbar',
2288
2296
'javelin-behavior-scrollbar',
2289
2297
'javelin-behavior-durable-column',
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
1986
1986
'PhabricatorChatLogQuery' => 'applications/chatlog/query/PhabricatorChatLogQuery.php',
1987
1987
'PhabricatorChunkedFileStorageEngine' => 'applications/files/engine/PhabricatorChunkedFileStorageEngine.php',
1988
1988
'PhabricatorClusterConfigOptions' => 'applications/config/option/PhabricatorClusterConfigOptions.php',
1989
+
'PhabricatorClusterDatabasesConfigOptionType' => 'infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php',
1989
1990
'PhabricatorColumnProxyInterface' => 'applications/project/interface/PhabricatorColumnProxyInterface.php',
1990
1991
'PhabricatorColumnsEditField' => 'applications/transactions/editfield/PhabricatorColumnsEditField.php',
1991
1992
'PhabricatorCommentEditEngineExtension' => 'applications/transactions/engineextension/PhabricatorCommentEditEngineExtension.php',
···
6392
6393
'PhabricatorChatLogQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
6393
6394
'PhabricatorChunkedFileStorageEngine' => 'PhabricatorFileStorageEngine',
6394
6395
'PhabricatorClusterConfigOptions' => 'PhabricatorApplicationConfigOptions',
6396
+
'PhabricatorClusterDatabasesConfigOptionType' => 'PhabricatorConfigJSONOptionType',
6395
6397
'PhabricatorColumnsEditField' => 'PhabricatorPHIDListEditField',
6396
6398
'PhabricatorCommentEditEngineExtension' => 'PhabricatorEditEngineExtension',
6397
6399
'PhabricatorCommentEditField' => 'PhabricatorEditField',
+6
src/applications/config/check/PhabricatorMySQLSetupCheck.php
+6
src/applications/config/check/PhabricatorMySQLSetupCheck.php
···
20
20
}
21
21
22
22
protected function executeChecks() {
23
+
// TODO: These checks should be executed against every reachable replica?
24
+
// See T10759.
25
+
if (PhabricatorEnv::isReadOnly()) {
26
+
return;
27
+
}
28
+
23
29
$max_allowed_packet = self::loadRawConfigValue('max_allowed_packet');
24
30
25
31
// This primarily supports setting the filesize limit for MySQL to 8MB,
+19
-1
src/applications/config/option/PhabricatorClusterConfigOptions.php
+19
-1
src/applications/config/option/PhabricatorClusterConfigOptions.php
···
20
20
}
21
21
22
22
public function getOptions() {
23
+
$databases_type = 'custom:PhabricatorClusterDatabasesConfigOptionType';
24
+
$databases_help = $this->deformat(pht(<<<EOTEXT
25
+
WARNING: This is a prototype option and the description below is currently pure
26
+
fantasy.
27
+
28
+
This option allows you to make Phabricator aware of database read replicas so
29
+
it can monitor database health, spread load, and degrade gracefully to
30
+
read-only mode in the event of a failure on the primary host. For help with
31
+
configuring cluster databases, see **[[ %s | %s ]]** in the documentation.
32
+
EOTEXT
33
+
,
34
+
PhabricatorEnv::getDoclink('Cluster: Databases'),
35
+
pht('Cluster: Databases')));
36
+
23
37
return array(
24
38
$this->newOption('cluster.addresses', 'list<string>', array())
25
39
->setLocked(true)
···
88
102
'into this mode automatically when it detects that the database '.
89
103
'master is unreachable, but you can activate it manually in '.
90
104
'order to perform maintenance or test configuration.')),
91
-
105
+
$this->newOption('cluster.databases', $databases_type, array())
106
+
->setHidden(true)
107
+
->setSummary(
108
+
pht('Configure database read replicas.'))
109
+
->setDescription($databases_help),
92
110
);
93
111
}
94
112
+3
src/docs/book/user.book
+3
src/docs/book/user.book
+40
src/docs/user/cluster/cluster.diviner
+40
src/docs/user/cluster/cluster.diviner
···
1
+
@title Clustering Introduction
2
+
@group cluster
3
+
4
+
Guide to configuring Phabricator across multiple hosts for availability and
5
+
performance.
6
+
7
+
Overview
8
+
========
9
+
10
+
WARNING: This feature is a very early prototype; the features this document
11
+
describes are mostly speculative fantasy.
12
+
13
+
Phabricator can be configured to run on mulitple hosts with redundant services
14
+
to improve its availability and scalability, and make disaster recovery much
15
+
easier.
16
+
17
+
Clustering is more complex to setup and maintain than running everything on a
18
+
single host, but greatly reduces the cost of recovering from hardware and
19
+
network failures.
20
+
21
+
Each Phabricator service has an array of clustering options that can be
22
+
configured independently. Configuring a cluster is inherently complex, and this
23
+
is an advanced feature aimed at installs with large userbases and experienced
24
+
operations personnel who need this high degree of flexibility.
25
+
26
+
The remainder of this document summarizes how to add redundancy to each
27
+
service and where your efforts are likely to have the greatest impact.
28
+
29
+
Cluster: Databases
30
+
=================
31
+
32
+
Configuring multiple database hosts is moderately complex, but normally has the
33
+
highest impact on availability and resistance to data loss. This is usually the
34
+
most important service to make redundant if your focus is on availability and
35
+
disaster recovery.
36
+
37
+
Configuring replicas allows Phabricator to run in read-only mode if you lose
38
+
the master, and to quickly promote the replica as a replacement.
39
+
40
+
For details, see @{article:Cluster: Databases}.
+161
src/docs/user/cluster/cluster_databases.diviner
+161
src/docs/user/cluster/cluster_databases.diviner
···
1
+
@title Cluster: Databases
2
+
@group intro
3
+
4
+
Configuring Phabricator to use multiple database hosts.
5
+
6
+
Overview
7
+
========
8
+
9
+
WARNING: This feature is a very early prototype; the features this document
10
+
describes are mostly speculative fantasy.
11
+
12
+
You can deploy Phabricator with multiple database hosts, configured as a master
13
+
and a set of replicas. The advantages of doing this are:
14
+
15
+
- faster recovery from disasters by promoting a replica;
16
+
- graceful degradation if the master fails;
17
+
- reduced load on the master; and
18
+
- some tools to help monitor and manage replica health.
19
+
20
+
This configuration is complex, and many installs do not need to pursue it.
21
+
22
+
Phabricator can not currently be configured into a multi-master mode, nor can
23
+
it be configured to automatically promote a replica to become the new master.
24
+
25
+
26
+
Setting up MySQL Replication
27
+
============================
28
+
29
+
TODO: Write this section.
30
+
31
+
32
+
Configuring Replicas
33
+
====================
34
+
35
+
Once your replicas are in working order, tell Phabricator about them by
36
+
configuring the `cluster.database` option. This option must be configured from
37
+
the command line or in configuration files because Phabricator needs to read
38
+
it //before// it can connect to databases.
39
+
40
+
This option value will list all of the database hosts that you want Phabricator
41
+
to interact with: your master and all your replicas. Each entry in the list
42
+
should have these keys:
43
+
44
+
- `host`: //Required string.// The database host name.
45
+
- `role`: //Required string.// The cluster role of this host, one of
46
+
`master` or `replica`.
47
+
- `port`: //Optional int.// The port to connect to. If omitted, the default
48
+
port from `mysql.port` will be used.
49
+
- `user`: //Optional string.// The MySQL username to use to connect to this
50
+
host. If omitted, the default from `mysql.user` will be used.
51
+
- `pass`: //Optional string.// The password to use to connect to this host.
52
+
If omitted, the default from `mysql.pass` will be used.
53
+
- `disabled`: //Optional bool.// If set to `true`, Phabricator will not
54
+
connect to this host. You can use this to temporarily take a host out
55
+
of service.
56
+
57
+
When `cluster.databases` is configured the `mysql.host` option is not used.
58
+
The other MySQL connection configuration options (`mysql.port`, `mysql.user`,
59
+
`mysql.pass`) are used only to provide defaults.
60
+
61
+
Once you've configured this option, restart Phabricator for the changes to take
62
+
effect, then continue to "Monitoring and Testing" to verify the configuration.
63
+
64
+
65
+
Monitoring and Testing
66
+
======================
67
+
68
+
TODO: Write this part.
69
+
70
+
Degradation to Read-Only Mode
71
+
=============================
72
+
73
+
Phabricator will degrade to read-only mode when any of these conditions occur:
74
+
75
+
- you turn it on explicitly;
76
+
- you configure cluster mode, but don't set up any masters;
77
+
- the master is misconfigured and unsafe to write to; or
78
+
- the master is unreachable.
79
+
80
+
When Phabricator is running in read-only mode, users can still read data and
81
+
browse and clone repositories, but they can not edit, update, or push new
82
+
changes. For example, users can still read disaster recovery information on
83
+
the wiki or emergency contact information on user profiles.
84
+
85
+
You can enable this mode explicitly by configuring `cluster.read-only`. Some
86
+
reasons you might want to do this include:
87
+
88
+
- to test that the mode works like you expect it to;
89
+
- to make sure that information you need will be available;
90
+
- to prevent new writes while performing database maintenance; or
91
+
- to permanently archive a Phabricator install.
92
+
93
+
You can also enable this mode implicitly by configuring `cluster.databases`
94
+
but disabling the master, or by not specifying any host as a master. This may
95
+
be more convenient than turning it on explicitly during the course of
96
+
operations work.
97
+
98
+
Before writing to a master, Phabricator will verify that the host is not
99
+
configured as a replica. This is a safety feature to prevent data loss if your
100
+
MySQL and Phabricator configurations disagree about replica configuration. If
101
+
your `master` is currently replicating from another host, Phabricator will
102
+
treat it as a `replica` instead and implicitly degrade into read-only mode.
103
+
104
+
Finally, if Phabricator is unable to reach the master, it will degrade into
105
+
read-only mode. For details on how Phabricator determines that a master is
106
+
unreachable, see "Unreachable Masters" below.
107
+
108
+
If a master becomes unreachable, this normally corresponds to loss of the
109
+
master host, a severed network link, or some other sort of disaster.
110
+
Phabricator will degrade and continue operating in read-only mode until the
111
+
master recovers or operations personnel can assess the situation and intervene.
112
+
113
+
If you end up in a situation where you have lost the master and can not get it
114
+
back online (or can not restore it quickly) you can promote a replica to become
115
+
the new master. See the next section, "Promoting a Replica", for details.
116
+
117
+
118
+
Promoting a Replica
119
+
===================
120
+
121
+
TODO: Write this, too.
122
+
123
+
124
+
Unreachable Masters
125
+
===================
126
+
127
+
This section describes how Phabricator determines that a master has been lost,
128
+
marks it unreachable, and degrades into read-only mode.
129
+
130
+
TODO: For now, it doesn't.
131
+
132
+
133
+
Backups
134
+
======
135
+
136
+
Even if you configure replication, you should still retain separate backup
137
+
snapshots. Replicas protect you from data loss if you lose a host, but they do
138
+
not let you recover from data mutation mistakes.
139
+
140
+
If something issues `DELETE` or `UPDATE` statements and destroys data on the
141
+
master, the mutation will propagate to the replicas almost immediately and the
142
+
data will be gone forever. Normally, the only way to recover this data is from
143
+
backup snapshots.
144
+
145
+
Although you should still have a backup process, your backup process can
146
+
safely pull dumps from a replica instead of the master. This operation can
147
+
be slow, so offloading it to a replica can make the perforance of the master
148
+
more consistent.
149
+
150
+
To dump from a replica, wait for this TODO to be resolved and then do whatever
151
+
it says to do:
152
+
153
+
TODO: Make `bin/storage dump` replica-aware. See T10758.
154
+
155
+
156
+
Next Steps
157
+
==========
158
+
159
+
Continue by:
160
+
161
+
- returning to @{article:Clustering Introduction}.
+98
src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php
+98
src/infrastructure/cluster/PhabricatorClusterDatabasesConfigOptionType.php
···
1
+
<?php
2
+
3
+
final class PhabricatorClusterDatabasesConfigOptionType
4
+
extends PhabricatorConfigJSONOptionType {
5
+
6
+
public function validateOption(PhabricatorConfigOption $option, $value) {
7
+
if (!is_array($value)) {
8
+
throw new Exception(
9
+
pht(
10
+
'Database cluster configuration is not valid: value must be a '.
11
+
'list of database hosts.'));
12
+
}
13
+
14
+
foreach ($value as $index => $spec) {
15
+
if (!is_array($spec)) {
16
+
throw new Exception(
17
+
pht(
18
+
'Database cluster configuration is not valid: each entry in the '.
19
+
'list must be a dictionary describing a database host, but '.
20
+
'the value with index "%s" is not a dictionary.',
21
+
$index));
22
+
}
23
+
}
24
+
25
+
$masters = array();
26
+
$map = array();
27
+
foreach ($value as $index => $spec) {
28
+
try {
29
+
PhutilTypeSpec::checkMap(
30
+
$spec,
31
+
array(
32
+
'host' => 'string',
33
+
'role' => 'string',
34
+
'port' => 'optional int',
35
+
'user' => 'optional string',
36
+
'pass' => 'optional string',
37
+
'disabled' => 'optional bool',
38
+
));
39
+
} catch (Exception $ex) {
40
+
throw new Exception(
41
+
pht(
42
+
'Database cluster configuration has an invalid host '.
43
+
'specification (at index "%s"): %s.',
44
+
$index,
45
+
$ex->getMessage()));
46
+
}
47
+
48
+
$role = $spec['role'];
49
+
$host = $spec['host'];
50
+
$port = idx($spec, 'port');
51
+
52
+
switch ($role) {
53
+
case 'master':
54
+
case 'replica':
55
+
break;
56
+
default:
57
+
throw new Exception(
58
+
pht(
59
+
'Database cluster configuration describes an invalid '.
60
+
'host ("%s", at index "%s") with an unrecognized role ("%s"). '.
61
+
'Valid roles are "%s" or "%s".',
62
+
$spec['host'],
63
+
$index,
64
+
$spec['role'],
65
+
'master',
66
+
'replica'));
67
+
}
68
+
69
+
if ($role === 'master') {
70
+
$masters[] = $host;
71
+
}
72
+
73
+
// We can't guarantee that you didn't just give the same host two
74
+
// different names in DNS, but this check can catch silly copy/paste
75
+
// mistakes.
76
+
$key = "{$host}:{$port}";
77
+
if (isset($map[$key])) {
78
+
throw new Exception(
79
+
pht(
80
+
'Database cluster configuration is invalid: it describes the '.
81
+
'same host ("%s") multiple times. Each host should appear only '.
82
+
'once in the list.',
83
+
$host));
84
+
}
85
+
$map[$key] = true;
86
+
}
87
+
88
+
if (count($masters) > 1) {
89
+
throw new Exception(
90
+
pht(
91
+
'Database cluster configuration is invalid: it describes multiple '.
92
+
'masters. No more than one host may be a master. Hosts currently '.
93
+
'configured as masters: %s.',
94
+
implode(', ', $masters)));
95
+
}
96
+
}
97
+
98
+
}
+3
-1
src/infrastructure/markup/PhabricatorMarkupEngine.php
+3
-1
src/infrastructure/markup/PhabricatorMarkupEngine.php
···
254
254
}
255
255
}
256
256
257
+
$is_readonly = PhabricatorEnv::isReadOnly();
258
+
257
259
foreach ($objects as $key => $info) {
258
260
// False check in case MySQL doesn't support unicode characters
259
261
// in the string (T1191), resulting in unserialize returning false.
···
279
281
->setCacheData($data)
280
282
->setMetadata($metadata);
281
283
282
-
if (isset($use_cache[$key])) {
284
+
if (isset($use_cache[$key]) && !$is_readonly) {
283
285
// This is just filling a cache and always safe, even on a read pathway.
284
286
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
285
287
$blocks[$key]->replace();
+1
-1
src/view/page/PhabricatorStandardPageView.php
+1
-1
src/view/page/PhabricatorStandardPageView.php