@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Provide `bin/storage quickstart` to automate generation of `quickstart.sql`
Summary:
Ref T1191. Currently, the `quickstart.sql` gets generated in a pretty manual fashion. This is a pain, and will become more of a pain in the world of utf8mb4.
Provide a workflow which does upgrade + adjust + dump + destroy, then massages the output to produce a workable `quickstart.sql`.
Test Plan: Inspected output; I'll test this more throughly before actually generating a new quickstart, but that's some ways away.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Maniphest Tasks: T1191
Differential Revision: https://secure.phabricator.com/D10603
+167
-33
+1
-1
resources/sql/patches/065.sshkeys.sql
+1
-1
resources/sql/patches/065.sshkeys.sql
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
2315
2315
'PhabricatorStorageManagementDestroyWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php',
2316
2316
'PhabricatorStorageManagementDumpWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php',
2317
2317
'PhabricatorStorageManagementProbeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php',
2318
+
'PhabricatorStorageManagementQuickstartWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php',
2318
2319
'PhabricatorStorageManagementStatusWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php',
2319
2320
'PhabricatorStorageManagementUpgradeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php',
2320
2321
'PhabricatorStorageManagementWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php',
···
5319
5320
'PhabricatorStorageManagementDestroyWorkflow' => 'PhabricatorStorageManagementWorkflow',
5320
5321
'PhabricatorStorageManagementDumpWorkflow' => 'PhabricatorStorageManagementWorkflow',
5321
5322
'PhabricatorStorageManagementProbeWorkflow' => 'PhabricatorStorageManagementWorkflow',
5323
+
'PhabricatorStorageManagementQuickstartWorkflow' => 'PhabricatorStorageManagementWorkflow',
5322
5324
'PhabricatorStorageManagementStatusWorkflow' => 'PhabricatorStorageManagementWorkflow',
5323
5325
'PhabricatorStorageManagementUpgradeWorkflow' => 'PhabricatorStorageManagementWorkflow',
5324
5326
'PhabricatorStorageManagementWorkflow' => 'PhabricatorManagementWorkflow',
+5
-25
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
+5
-25
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
···
149
149
150
150
$api = $this->getAPI();
151
151
152
-
if ($api->isCharacterSetAvailable('utf8mb4')) {
153
-
// If utf8mb4 is available, we use it with the utf8mb4_unicode_ci
154
-
// collation. This is most correct, and will sort properly.
155
-
156
-
$utf8_charset = 'utf8mb4';
157
-
$utf8_binary_collation = 'utf8mb4_bin';
158
-
$utf8_sorting_collation = 'utf8mb4_unicode_ci';
159
-
} else {
160
-
// If utf8mb4 is not available, we use binary. This allows us to store
161
-
// 4-byte unicode characters. This has some tradeoffs:
162
-
//
163
-
// Unicode characters won't sort correctly. There's nothing we can do
164
-
// about this while still supporting 4-byte characters.
165
-
//
166
-
// It's possible that strings will be truncated in the middle of a
167
-
// character on insert. We encourage users to set STRICT_ALL_TABLES
168
-
// to prevent this.
169
-
170
-
$utf8_charset = 'binary';
171
-
$utf8_binary_collation = 'binary';
172
-
$utf8_sorting_collation = 'binary';
173
-
}
152
+
$charset_info = $api->getCharsetInfo();
153
+
list($charset, $collate_text, $collate_sort) = $charset_info;
174
154
175
155
$specs = id(new PhutilSymbolLoader())
176
156
->setAncestorClass('PhabricatorConfigSchemaSpec')
···
179
159
$server_schema = new PhabricatorConfigServerSchema();
180
160
foreach ($specs as $spec) {
181
161
$spec
182
-
->setUTF8Charset($utf8_charset)
183
-
->setUTF8BinaryCollation($utf8_binary_collation)
184
-
->setUTF8SortingCollation($utf8_sorting_collation)
162
+
->setUTF8Charset($charset)
163
+
->setUTF8BinaryCollation($collate_text)
164
+
->setUTF8SortingCollation($collate_sort)
185
165
->setServer($server_schema)
186
166
->buildSchemata($server_schema);
187
167
}
+1
-2
src/applications/fact/storage/PhabricatorFactAggregate.php
+1
-2
src/applications/fact/storage/PhabricatorFactAggregate.php
···
9
9
public function getConfiguration() {
10
10
return array(
11
11
self::CONFIG_COLUMN_SCHEMA => array(
12
-
'id' => null,
12
+
'id' => 'id64',
13
13
'factType' => 'text32',
14
14
'valueX' => 'uint64',
15
15
),
16
16
self::CONFIG_KEY_SCHEMA => array(
17
-
'PRIMARY' => null,
18
17
'factType' => array(
19
18
'columns' => array('factType', 'objectPHID'),
20
19
'unique' => true,
+1
-2
src/applications/fact/storage/PhabricatorFactRaw.php
+1
-2
src/applications/fact/storage/PhabricatorFactRaw.php
···
15
15
public function getConfiguration() {
16
16
return array(
17
17
self::CONFIG_COLUMN_SCHEMA => array(
18
-
'id' => null,
18
+
'id' => 'id64',
19
19
'factType' => 'text32',
20
20
'objectA' => 'phid',
21
21
'valueX' => 'sint64',
22
22
'valueY' => 'sint64',
23
23
),
24
24
self::CONFIG_KEY_SCHEMA => array(
25
-
'PRIMARY' => null,
26
25
'objectPHID' => array(
27
26
'columns' => array('objectPHID'),
28
27
),
+33
src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
+33
src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
···
182
182
183
183
$conn = $this->getConn(null);
184
184
185
+
$charset_info = $this->getCharsetInfo();
186
+
list($charset, $collate_text, $collate_sort) = $charset_info;
187
+
185
188
foreach ($queries as $query) {
186
189
$query = str_replace('{$NAMESPACE}', $this->namespace, $query);
190
+
$query = str_replace('{$CHARSET}', $charset, $query);
191
+
$query = str_replace('{$COLLATE_TEXT}', $collate_text, $query);
192
+
$query = str_replace('{$COLLATE_SORT}', $collate_sort, $query);
187
193
queryfx(
188
194
$conn,
189
195
'%Q',
···
206
212
$character_set);
207
213
208
214
return (bool)$result;
215
+
}
216
+
217
+
public function getCharsetInfo() {
218
+
if ($this->isCharacterSetAvailable('utf8mb4')) {
219
+
// If utf8mb4 is available, we use it with the utf8mb4_unicode_ci
220
+
// collation. This is most correct, and will sort properly.
221
+
222
+
$charset = 'utf8mb4';
223
+
$collate_text = 'utf8mb4_bin';
224
+
$collate_sort = 'utf8mb4_unicode_ci';
225
+
} else {
226
+
// If utf8mb4 is not available, we use binary. This allows us to store
227
+
// 4-byte unicode characters. This has some tradeoffs:
228
+
//
229
+
// Unicode characters won't sort correctly. There's nothing we can do
230
+
// about this while still supporting 4-byte characters.
231
+
//
232
+
// It's possible that strings will be truncated in the middle of a
233
+
// character on insert. We encourage users to set STRICT_ALL_TABLES
234
+
// to prevent this.
235
+
236
+
$charset = 'binary';
237
+
$collate_text = 'binary';
238
+
$collate_sort = 'binary';
239
+
}
240
+
241
+
return array($charset, $collate_text, $collate_sort);
209
242
}
210
243
211
244
}
+2
-3
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php
+2
-3
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php
···
17
17
$force = $args->getArg('force');
18
18
19
19
$this->requireAllPatchesApplied();
20
-
$this->adjustSchemata($force);
21
-
return 0;
20
+
return $this->adjustSchemata($force);
22
21
}
23
22
24
23
private function requireAllPatchesApplied() {
···
258
257
$console->writeOut(
259
258
"%s\n",
260
259
pht('Completed fixing all schema issues.'));
261
-
return;
260
+
return 0;
262
261
}
263
262
264
263
$table = id(new PhutilConsoleTable())
+122
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php
+122
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhabricatorStorageManagementQuickstartWorkflow
4
+
extends PhabricatorStorageManagementWorkflow {
5
+
6
+
public function didConstruct() {
7
+
$this
8
+
->setName('quickstart')
9
+
->setExamples('**quickstart** [__options__]')
10
+
->setSynopsis(
11
+
pht(
12
+
'Generate a new quickstart database dump. This command is mostly '.
13
+
'useful when developing Phabricator.'))
14
+
->setArguments(
15
+
array(
16
+
array(
17
+
'name' => 'output',
18
+
'param' => 'file',
19
+
'help' => pht('Specify output file to write.'),
20
+
),
21
+
));
22
+
}
23
+
24
+
public function execute(PhutilArgumentParser $args) {
25
+
$output = $args->getArg('output');
26
+
if (!$output) {
27
+
throw new PhutilArgumentUsageException(
28
+
pht(
29
+
'Specify a file to write with `--output`.'));
30
+
}
31
+
32
+
$namespace = 'phabricator_quickstart_'.Filesystem::readRandomCharacters(8);
33
+
34
+
$bin = dirname(phutil_get_library_root('phabricator')).'/bin/storage';
35
+
36
+
$err = phutil_passthru(
37
+
'%s upgrade --force --no-quickstart --namespace %s',
38
+
$bin,
39
+
$namespace);
40
+
if ($err) {
41
+
return $err;
42
+
}
43
+
44
+
$err = phutil_passthru(
45
+
'%s adjust --force --namespace %s',
46
+
$bin,
47
+
$namespace);
48
+
if ($err) {
49
+
return $err;
50
+
}
51
+
52
+
$tmp = new TempFile();
53
+
$err = phutil_passthru(
54
+
'%s dump --namespace %s > %s',
55
+
$bin,
56
+
$namespace,
57
+
$tmp);
58
+
if ($err) {
59
+
return $err;
60
+
}
61
+
62
+
$err = phutil_passthru(
63
+
'%s destroy --force --namespace %s',
64
+
$bin,
65
+
$namespace);
66
+
if ($err) {
67
+
return $err;
68
+
}
69
+
70
+
$dump = Filesystem::readFile($tmp);
71
+
72
+
$dump = str_replace(
73
+
$namespace,
74
+
'{$NAMESPACE}',
75
+
$dump);
76
+
77
+
$dump = str_replace(
78
+
'utf8mb4_bin',
79
+
'{$COLLATE_TEXT}',
80
+
$dump);
81
+
82
+
$dump = str_replace(
83
+
'utf8mb4_unicode_ci',
84
+
'{$COLLATE_SORT}',
85
+
$dump);
86
+
87
+
$dump = str_replace(
88
+
'utf8mb4',
89
+
'{$CHARSET}',
90
+
$dump);
91
+
92
+
// Strip out a bunch of unnecessary commands which make the dump harder
93
+
// to handle and slower to import.
94
+
95
+
// Remove character set adjustments and key disables.
96
+
$dump = preg_replace(
97
+
'(^/\*.*\*/;$)m',
98
+
'',
99
+
$dump);
100
+
101
+
// Remove comments.
102
+
$dump = preg_replace('/^--.*$/m', '', $dump);
103
+
104
+
// Remove table drops, locks, and unlocks. These are never relevant when
105
+
// performing q quickstart.
106
+
$dump = preg_replace(
107
+
'/^(DROP TABLE|LOCK TABLES|UNLOCK TABLES).*$/m',
108
+
'',
109
+
$dump);
110
+
111
+
// Collapse adjacent newlines.
112
+
$dump = preg_replace('/\n\s*\n/', "\n", $dump);
113
+
114
+
$dump = str_replace(';', ";\n", $dump);
115
+
$dump = trim($dump)."\n";
116
+
117
+
Filesystem::writeFile($output, $dump);
118
+
119
+
return 0;
120
+
}
121
+
122
+
}