@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Detect replicating masters and fatal (also, warn on nonreplicating replicas)
Summary:
Ref T10759. Check master/replica status during startup.
After D16903, this also means that we check this status after a database comes back online after being unreachable.
If a master is replicating, fatal (since this can do a million kinds of bad things).
If a replica is not replicating, warn (this just means the replica is behind so some data is at risk).
Also: if your masters were actually configured properly (mine weren't until this change detected it), we would throw away patches as we applied them, so they would only apply to the //first// master. Instead, properly apply all migration patches to all masters.
Test Plan:
- Started Phabricator with a replicating master, got a fatal.
- Stopped replication on a replica, got a warning.
- With two non-replicating masters, upgraded storage.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T10759
Differential Revision: https://secure.phabricator.com/D16904
+56
-13
+43
-1
src/applications/config/check/PhabricatorDatabaseSetupCheck.php
+43
-1
src/applications/config/check/PhabricatorDatabaseSetupCheck.php
···
43
43
$port));
44
44
}
45
45
46
-
$refs = PhabricatorDatabaseRef::getActiveDatabaseRefs();
46
+
$refs = PhabricatorDatabaseRef::queryAll();
47
47
$refs = mpull($refs, null, 'getRefKey');
48
48
49
49
// Test if we can connect to each database first. If we can not connect
···
164
164
165
165
return true;
166
166
}
167
+
168
+
// NOTE: It's possible that replication is broken but we have not been
169
+
// granted permission to "SHOW SLAVE STATUS" so we can't figure it out.
170
+
// We allow this kind of configuration and survive these checks, trusting
171
+
// that operations knows what they're doing. This issue is shown on the
172
+
// "Database Servers" console.
173
+
174
+
switch ($ref->getReplicaStatus()) {
175
+
case PhabricatorDatabaseRef::REPLICATION_MASTER_REPLICA:
176
+
$message = pht(
177
+
'Database host "%s" is configured as a master, but is replicating '.
178
+
'another host. This is dangerous and can mangle or destroy data. '.
179
+
'Only replicas should be replicating. Stop replication on the '.
180
+
'host or reconfigure Phabricator.',
181
+
$ref->getRefKey());
182
+
183
+
$this->newIssue('db.master.replicating')
184
+
->setName(pht('Replicating Master'))
185
+
->setIsFatal(true)
186
+
->setMessage($message);
187
+
188
+
return true;
189
+
case PhabricatorDatabaseRef::REPLICATION_REPLICA_NONE:
190
+
case PhabricatorDatabaseRef::REPLICATION_NOT_REPLICATING:
191
+
if (!$ref->getIsMaster()) {
192
+
$message = pht(
193
+
'Database replica "%s" is listed as a replica, but is not '.
194
+
'currently replicating. You are vulnerable to data loss if '.
195
+
'the master fails.',
196
+
$ref->getRefKey());
197
+
198
+
// This isn't a fatal because it can normally only put data at risk,
199
+
// not actually do anything destructive or unrecoverable.
200
+
201
+
$this->newIssue('db.replica.not-replicating')
202
+
->setName(pht('Nonreplicating Replica'))
203
+
->setMessage($message);
204
+
}
205
+
break;
206
+
}
207
+
208
+
167
209
}
168
210
}
+4
-5
src/infrastructure/cluster/PhabricatorDatabaseRef.php
+4
-5
src/infrastructure/cluster/PhabricatorDatabaseRef.php
···
308
308
}
309
309
310
310
public static function queryAll() {
311
-
$refs = self::newRefs();
311
+
$refs = self::getActiveDatabaseRefs();
312
+
return self::queryRefs($refs);
313
+
}
312
314
315
+
private static function queryRefs(array $refs) {
313
316
foreach ($refs as $ref) {
314
-
if ($ref->getDisabled()) {
315
-
continue;
316
-
}
317
-
318
317
$conn = $ref->newManagementConnection();
319
318
320
319
$t_start = microtime(true);
+9
-7
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
+9
-7
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
···
842
842
$no_quickstart,
843
843
$init_only) {
844
844
845
+
$patches = $this->patches;
846
+
845
847
$applied = $api->getAppliedPatches();
846
848
if ($applied === null) {
847
849
if ($this->dryRun) {
···
864
866
// adjustment phase.
865
867
$this->didInitialize = true;
866
868
867
-
$legacy = $api->getLegacyPatches($this->patches);
869
+
$legacy = $api->getLegacyPatches($patches);
868
870
if ($legacy || $no_quickstart || $init_only) {
869
871
870
872
// If we have legacy patches, we can't quickstart.
···
921
923
922
924
while (true) {
923
925
$applied_something = false;
924
-
foreach ($this->patches as $key => $patch) {
926
+
foreach ($patches as $key => $patch) {
925
927
if (isset($applied[$key])) {
926
-
unset($this->patches[$key]);
928
+
unset($patches[$key]);
927
929
continue;
928
930
}
929
931
930
932
if ($apply_only && $apply_only != $key) {
931
-
unset($this->patches[$key]);
933
+
unset($patches[$key]);
932
934
continue;
933
935
}
934
936
···
968
970
}
969
971
}
970
972
971
-
unset($this->patches[$key]);
973
+
unset($patches[$key]);
972
974
$applied[$key] = true;
973
975
}
974
976
975
977
if (!$applied_something) {
976
-
if (count($this->patches)) {
978
+
if (count($patches)) {
977
979
throw new Exception(
978
980
pht(
979
981
'Some patches could not be applied to "%s": %s',
980
982
$api->getRef()->getRefKey(),
981
-
implode(', ', array_keys($this->patches))));
983
+
implode(', ', array_keys($patches))));
982
984
} else if (!$this->dryRun && !$apply_only) {
983
985
echo pht(
984
986
'Storage is up to date on "%s". Use "%s" for details.',