@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Run "DatabaseSetup" checks against all configured hosts
Summary:
Ref T10759. Currently, these checks run only against configured masters. Instead, check every host.
These checks also sort of cheat through restart during a recovery, when some hosts will be unreachable: they test for "disaster" by seeing if no masters are reachable, and just skip all the checks in that case.
This is bad for at least two reasons:
- After recent changes, it is possible that //some// masters are dead but it's still OK to start. For example, "slowvote" may have no master, but everything else is reachable. We can safely run without slowvote.
- It's possible to start during a disaster and miss important setup checks completely, since we skip them, get a clean bill of health, and never re-test them.
Instead:
- Test each host individually.
- Fundamental problems (lack of InnoDB, bad schema) are fatal on any host.
- If we can't connect, raise it as a //warning// to make sure we check it later. If you start during a disaster, we still want to make sure that schemata are up to date if you later recover a host.
In particular, I'm going to add these checks soon:
- Fatal if a "master" is replicating.
- Fatal if a "replica" is not replicating.
- Fatal if a database partition config differs from web partition config.
- When we let a database off with a warning because it's down, and later upgrade it to a fatal because we discover it is broken after it comes up again, fatal everything. Currently, we keep running if we "discover" the presence of new fatals after surviving setup checks for the first time.
Test Plan:
- Configured with multiple masters, intentionally broke one (simulating a disaster where one master is lost), saw Phabricator still startup.
- Tested individual setup checks by intentionally breaking them.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T10759
Differential Revision: https://secure.phabricator.com/D16902
+117
-75
+2
-1
src/aphront/configuration/AphrontApplicationConfiguration.php
+2
-1
src/aphront/configuration/AphrontApplicationConfiguration.php
···
89
89
90
90
if ($database_exception) {
91
91
$issue = PhabricatorSetupIssue::newDatabaseConnectionIssue(
92
-
$database_exception);
92
+
$database_exception,
93
+
true);
93
94
$response = PhabricatorSetupCheck::newIssueResponse($issue);
94
95
return self::writeResponse($sink, $response);
95
96
}
+66
-37
src/applications/config/check/PhabricatorDatabaseSetupCheck.php
+66
-37
src/applications/config/check/PhabricatorDatabaseSetupCheck.php
···
43
43
$port));
44
44
}
45
45
46
-
$masters = PhabricatorDatabaseRef::getMasterDatabaseRefs();
47
-
if (!$masters) {
48
-
// If we're implicitly in read-only mode during disaster recovery,
49
-
// don't bother with these setup checks.
50
-
return;
51
-
}
46
+
$refs = PhabricatorDatabaseRef::getActiveDatabaseRefs();
47
+
$refs = mpull($refs, null, 'getRefKey');
52
48
53
-
foreach ($masters as $master) {
54
-
if ($this->checkMasterDatabase($master)) {
55
-
break;
49
+
// Test if we can connect to each database first. If we can not connect
50
+
// to a particular database, we only raise a warning: this allows new web
51
+
// nodes to start during a disaster, when some databases may be correctly
52
+
// configured but not reachable.
53
+
54
+
$connect_map = array();
55
+
$any_connection = false;
56
+
foreach ($refs as $ref_key => $ref) {
57
+
$conn_raw = $ref->newManagementConnection();
58
+
59
+
try {
60
+
queryfx($conn_raw, 'SELECT 1');
61
+
$database_exception = null;
62
+
$any_connection = true;
63
+
} catch (AphrontInvalidCredentialsQueryException $ex) {
64
+
$database_exception = $ex;
65
+
} catch (AphrontConnectionQueryException $ex) {
66
+
$database_exception = $ex;
67
+
}
68
+
69
+
if ($database_exception) {
70
+
$connect_map[$ref_key] = $database_exception;
71
+
unset($refs[$ref_key]);
56
72
}
57
73
}
58
-
}
59
74
60
-
private function checkMasterDatabase(PhabricatorDatabaseRef $master) {
61
-
$conn_raw = $master->newManagementConnection();
75
+
if ($connect_map) {
76
+
// This is only a fatal error if we could not connect to anything. If
77
+
// possible, we still want to start if some database hosts can not be
78
+
// reached.
79
+
$is_fatal = !$any_connection;
62
80
63
-
try {
64
-
queryfx($conn_raw, 'SELECT 1');
65
-
$database_exception = null;
66
-
} catch (AphrontInvalidCredentialsQueryException $ex) {
67
-
$database_exception = $ex;
68
-
} catch (AphrontConnectionQueryException $ex) {
69
-
$database_exception = $ex;
81
+
foreach ($connect_map as $ref_key => $database_exception) {
82
+
$issue = PhabricatorSetupIssue::newDatabaseConnectionIssue(
83
+
$database_exception,
84
+
$is_fatal);
85
+
$this->addIssue($issue);
86
+
}
70
87
}
71
88
72
-
if ($database_exception) {
73
-
$issue = PhabricatorSetupIssue::newDatabaseConnectionIssue(
74
-
$database_exception);
75
-
$this->addIssue($issue);
76
-
return true;
89
+
foreach ($refs as $ref_key => $ref) {
90
+
if ($this->executeRefChecks($ref)) {
91
+
return;
92
+
}
77
93
}
94
+
}
95
+
96
+
private function executeRefChecks(PhabricatorDatabaseRef $ref) {
97
+
$conn_raw = $ref->newManagementConnection();
98
+
$ref_key = $ref->getRefKey();
78
99
79
100
$engines = queryfx_all($conn_raw, 'SHOW ENGINES');
80
101
$engines = ipull($engines, 'Support', 'Engine');
···
82
103
$innodb = idx($engines, 'InnoDB');
83
104
if ($innodb != 'YES' && $innodb != 'DEFAULT') {
84
105
$message = pht(
85
-
"The 'InnoDB' engine is not available in MySQL. Enable InnoDB in ".
86
-
"your MySQL configuration.".
106
+
'The "InnoDB" engine is not available in MySQL (on host "%s"). '.
107
+
'Enable InnoDB in your MySQL configuration.'.
87
108
"\n\n".
88
-
"(If you aleady created tables, MySQL incorrectly used some other ".
89
-
"engine to create them. You need to convert them or drop and ".
90
-
"reinitialize them.)");
109
+
'(If you aleady created tables, MySQL incorrectly used some other '.
110
+
'engine to create them. You need to convert them or drop and '.
111
+
'reinitialize them.)',
112
+
$ref_key);
91
113
92
114
$this->newIssue('mysql.innodb')
93
115
->setName(pht('MySQL InnoDB Engine Not Available'))
94
116
->setMessage($message)
95
117
->setIsFatal(true);
118
+
96
119
return true;
97
120
}
98
121
···
103
126
104
127
if (empty($databases[$namespace.'_meta_data'])) {
105
128
$message = pht(
106
-
"Run the storage upgrade script to setup Phabricator's database ".
107
-
"schema.");
129
+
'Run the storage upgrade script to setup databases (host "%s" has '.
130
+
'not been initialized).',
131
+
$ref_key);
108
132
109
133
$this->newIssue('storage.upgrade')
110
134
->setName(pht('Setup MySQL Schema'))
111
135
->setMessage($message)
112
136
->setIsFatal(true)
113
137
->addCommand(hsprintf('<tt>phabricator/ $</tt> ./bin/storage upgrade'));
138
+
114
139
return true;
115
140
}
116
141
117
-
$conn_meta = $master->newApplicationConnection(
142
+
$conn_meta = $ref->newApplicationConnection(
118
143
$namespace.'_meta_data');
119
144
120
145
$applied = queryfx_all($conn_meta, 'SELECT patch FROM patch_status');
···
124
149
$diff = array_diff_key($all, $applied);
125
150
126
151
if ($diff) {
152
+
$message = pht(
153
+
'Run the storage upgrade script to upgrade databases (host "%s" is '.
154
+
'out of date). Missing patches: %s.',
155
+
$ref_key,
156
+
implode(', ', array_keys($diff)));
157
+
127
158
$this->newIssue('storage.patch')
128
159
->setName(pht('Upgrade MySQL Schema'))
129
-
->setMessage(
130
-
pht(
131
-
"Run the storage upgrade script to upgrade Phabricator's ".
132
-
"database schema. Missing patches:<br />%s<br />",
133
-
phutil_implode_html(phutil_tag('br'), array_keys($diff))))
160
+
->setIsFatal(true)
161
+
->setMessage($message)
134
162
->addCommand(
135
163
hsprintf('<tt>phabricator/ $</tt> ./bin/storage upgrade'));
164
+
136
165
return true;
137
166
}
138
167
}
+10
-3
src/applications/config/issue/PhabricatorSetupIssue.php
+10
-3
src/applications/config/issue/PhabricatorSetupIssue.php
···
21
21
private $links;
22
22
23
23
public static function newDatabaseConnectionIssue(
24
-
AphrontQueryException $ex) {
24
+
AphrontQueryException $ex,
25
+
$is_fatal) {
25
26
26
27
$message = pht(
27
28
"Unable to connect to MySQL!\n\n".
···
29
30
"Make sure Phabricator and MySQL are correctly configured.",
30
31
$ex->getMessage());
31
32
32
-
return id(new self())
33
+
$issue = id(new self())
33
34
->setIssueKey('mysql.connect')
34
35
->setName(pht('Can Not Connect to MySQL'))
35
36
->setMessage($message)
36
-
->setIsFatal(true)
37
+
->setIsFatal($is_fatal)
37
38
->addRelatedPhabricatorConfig('mysql.host')
38
39
->addRelatedPhabricatorConfig('mysql.port')
39
40
->addRelatedPhabricatorConfig('mysql.user')
40
41
->addRelatedPhabricatorConfig('mysql.pass');
42
+
43
+
if (PhabricatorEnv::getEnvConfig('cluster.databases')) {
44
+
$issue->addRelatedPhabricatorConfig('cluster.databases');
45
+
}
46
+
47
+
return $issue;
41
48
}
42
49
43
50
public function addCommand($command) {
+39
-34
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
+39
-34
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
···
23
23
public function didExecute(PhutilArgumentParser $args) {
24
24
$console = PhutilConsole::getConsole();
25
25
26
-
$api = $this->getSingleAPI();
27
-
28
26
if (!$this->isDryRun() && !$this->isForce()) {
29
27
$console->writeOut(
30
28
phutil_console_wrap(
···
44
42
}
45
43
}
46
44
47
-
$patches = $this->getPatches();
45
+
$apis = $this->getMasterAPIs();
46
+
foreach ($apis as $api) {
47
+
$patches = $this->getPatches();
48
+
49
+
if ($args->getArg('unittest-fixtures')) {
50
+
$conn = $api->getConn(null);
51
+
$databases = queryfx_all(
52
+
$conn,
53
+
'SELECT DISTINCT(TABLE_SCHEMA) AS db '.
54
+
'FROM INFORMATION_SCHEMA.TABLES '.
55
+
'WHERE TABLE_SCHEMA LIKE %>',
56
+
PhabricatorTestCase::NAMESPACE_PREFIX);
57
+
$databases = ipull($databases, 'db');
58
+
} else {
59
+
$databases = $api->getDatabaseList($patches);
60
+
$databases[] = $api->getDatabaseName('meta_data');
48
61
49
-
if ($args->getArg('unittest-fixtures')) {
50
-
$conn = $api->getConn(null);
51
-
$databases = queryfx_all(
52
-
$conn,
53
-
'SELECT DISTINCT(TABLE_SCHEMA) AS db '.
54
-
'FROM INFORMATION_SCHEMA.TABLES '.
55
-
'WHERE TABLE_SCHEMA LIKE %>',
56
-
PhabricatorTestCase::NAMESPACE_PREFIX);
57
-
$databases = ipull($databases, 'db');
58
-
} else {
59
-
$databases = $api->getDatabaseList($patches);
60
-
$databases[] = $api->getDatabaseName('meta_data');
62
+
// These are legacy databases that were dropped long ago. See T2237.
63
+
$databases[] = $api->getDatabaseName('phid');
64
+
$databases[] = $api->getDatabaseName('directory');
65
+
}
61
66
62
-
// These are legacy databases that were dropped long ago. See T2237.
63
-
$databases[] = $api->getDatabaseName('phid');
64
-
$databases[] = $api->getDatabaseName('directory');
65
-
}
67
+
foreach ($databases as $database) {
68
+
if ($this->isDryRun()) {
69
+
$console->writeOut(
70
+
"%s\n",
71
+
pht("DRYRUN: Would drop database '%s'.", $database));
72
+
} else {
73
+
$console->writeOut(
74
+
"%s\n",
75
+
pht("Dropping database '%s'...", $database));
76
+
queryfx(
77
+
$api->getConn(null),
78
+
'DROP DATABASE IF EXISTS %T',
79
+
$database);
80
+
}
81
+
}
66
82
67
-
foreach ($databases as $database) {
68
-
if ($this->isDryRun()) {
83
+
if (!$this->isDryRun()) {
69
84
$console->writeOut(
70
85
"%s\n",
71
-
pht("DRYRUN: Would drop database '%s'.", $database));
72
-
} else {
73
-
$console->writeOut(
74
-
"%s\n",
75
-
pht("Dropping database '%s'...", $database));
76
-
queryfx(
77
-
$api->getConn(null),
78
-
'DROP DATABASE IF EXISTS %T',
79
-
$database);
86
+
pht(
87
+
'Storage on "%s" was destroyed.',
88
+
$api->getRef()->getRefKey()));
80
89
}
81
-
}
82
-
83
-
if (!$this->isDryRun()) {
84
-
$console->writeOut("%s\n", pht('Storage was destroyed.'));
85
90
}
86
91
87
92
return 0;