@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Introduce storage patch "phases" to allow index-rebuilding patches to execute after worker queue schema changes
Summary:
Ref T13591. Some storage patches queue worker tasks, currently always to rebuild search indexes.
These patches can not execute in creation order if a later patch modifies the worker task table, since they'll try to perform a modern INSERT against an out-of-date table schema. Such a modification is desirable in the context of T13591, but making it causes these patches to fail.
Patches have an existing "after" mechanism which allows them to have explicit dependencies. This mechanism could be used to resolve this issue, but all patches with a dependency like this would need to be updated every time the queue table changes.
Instead, introduce "phases" to provide broader ordering rules. There are now two phases: "default" and "worker". Patches in the "worker" phase execute after patches in the "default" phase.
Phases may eventually be further separated, but
Test Plan:
- Ran `bin/storage status`, saw patches annotated with phases.
- Will apply `containerPHID` changes on top of this.
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13591
Differential Revision: https://secure.phabricator.com/D21529
+215
-17
+2
resources/sql/autopatches/20190412.dashboard.13.rebuild.php
+2
resources/sql/autopatches/20190412.dashboard.13.rebuild.php
+2
resources/sql/autopatches/20190412.herald.01.rebuild.php
+2
resources/sql/autopatches/20190412.herald.01.rebuild.php
+2
resources/sql/autopatches/20190909.herald.01.rebuild.php
+2
resources/sql/autopatches/20190909.herald.01.rebuild.php
+2
resources/sql/autopatches/20191028.uriindex.01.rebuild.php
+2
resources/sql/autopatches/20191028.uriindex.01.rebuild.php
+36
src/infrastructure/storage/management/PhabricatorStoragePatch.php
+36
src/infrastructure/storage/management/PhabricatorStoragePatch.php
···
9
9
private $after;
10
10
private $legacy;
11
11
private $dead;
12
+
private $phase;
13
+
14
+
const PHASE_DEFAULT = 'default';
15
+
const PHASE_WORKER = 'worker';
12
16
13
17
public function __construct(array $dict) {
14
18
$this->key = $dict['key'];
···
18
22
$this->name = $dict['name'];
19
23
$this->after = $dict['after'];
20
24
$this->dead = $dict['dead'];
25
+
$this->phase = $dict['phase'];
21
26
}
22
27
23
28
public function getLegacy() {
···
42
47
43
48
public function getKey() {
44
49
return $this->key;
50
+
}
51
+
52
+
public function getPhase() {
53
+
return $this->phase;
45
54
}
46
55
47
56
public function isDead() {
···
50
59
51
60
public function getIsGlobalPatch() {
52
61
return ($this->getType() == 'php');
62
+
}
63
+
64
+
public static function getPhaseList() {
65
+
return array_keys(self::getPhaseMap());
66
+
}
67
+
68
+
public static function getDefaultPhase() {
69
+
return self::PHASE_DEFAULT;
70
+
}
71
+
72
+
private static function getPhaseMap() {
73
+
return array(
74
+
self::PHASE_DEFAULT => array(
75
+
'order' => 0,
76
+
),
77
+
self::PHASE_WORKER => array(
78
+
'order' => 1,
79
+
),
80
+
);
81
+
}
82
+
83
+
public function newSortVector() {
84
+
$map = self::getPhaseMap();
85
+
$phase = $this->getPhase();
86
+
87
+
return id(new PhutilSortVector())
88
+
->addInt($map[$phase]['order']);
53
89
}
54
90
55
91
}
+17
-10
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php
+17
-10
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php
···
33
33
$table = id(new PhutilConsoleTable())
34
34
->setShowHeader(false)
35
35
->addColumn('id', array('title' => pht('ID')))
36
+
->addColumn('phase', array('title' => pht('Phase')))
36
37
->addColumn('host', array('title' => pht('Host')))
37
38
->addColumn('status', array('title' => pht('Status')))
38
39
->addColumn('duration', array('title' => pht('Duration')))
···
49
50
$duration = pht('%s us', new PhutilNumber($duration));
50
51
}
51
52
52
-
$table->addRow(array(
53
-
'id' => $patch->getFullKey(),
54
-
'host' => $ref->getRefKey(),
55
-
'status' => in_array($patch->getFullKey(), $applied)
56
-
? pht('Applied')
57
-
: pht('Not Applied'),
58
-
'duration' => $duration,
59
-
'type' => $patch->getType(),
60
-
'name' => $patch->getName(),
61
-
));
53
+
if (in_array($patch->getFullKey(), $applied)) {
54
+
$status = pht('Applied');
55
+
} else {
56
+
$status = pht('Not Applied');
57
+
}
58
+
59
+
$table->addRow(
60
+
array(
61
+
'id' => $patch->getFullKey(),
62
+
'phase' => $patch->getPhase(),
63
+
'host' => $ref->getRefKey(),
64
+
'status' => $status,
65
+
'duration' => $duration,
66
+
'type' => $patch->getType(),
67
+
'name' => $patch->getName(),
68
+
));
62
69
}
63
70
64
71
$table->draw();
+4
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
+4
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
···
922
922
$patches = $this->patches;
923
923
$is_dryrun = $this->dryRun;
924
924
925
+
// We expect that patches should already be sorted properly. However,
926
+
// phase behavior will be wrong if they aren't, so make sure.
927
+
$patches = msortv($patches, 'newSortVector');
928
+
925
929
$api_map = array();
926
930
foreach ($apis as $api) {
927
931
$api_map[$api->getRef()->getRefKey()] = $api;
+150
-7
src/infrastructure/storage/patch/PhabricatorSQLPatchList.php
+150
-7
src/infrastructure/storage/patch/PhabricatorSQLPatchList.php
···
27
27
$directory));
28
28
}
29
29
30
+
$patch_type = $matches[1];
31
+
$patch_full_path = rtrim($directory, '/').'/'.$patch;
32
+
33
+
$attributes = array();
34
+
if ($patch_type === 'php') {
35
+
$attributes = $this->getPHPPatchAttributes(
36
+
$patch,
37
+
$patch_full_path);
38
+
}
39
+
30
40
$patches[$patch] = array(
31
-
'type' => $matches[1],
32
-
'name' => rtrim($directory, '/').'/'.$patch,
33
-
);
41
+
'type' => $patch_type,
42
+
'name' => $patch_full_path,
43
+
) + $attributes;
34
44
}
35
45
36
46
return $patches;
···
45
55
$specs = array();
46
56
$seen_namespaces = array();
47
57
58
+
$phases = PhabricatorStoragePatch::getPhaseList();
59
+
$phases = array_fuse($phases);
60
+
61
+
$default_phase = PhabricatorStoragePatch::getDefaultPhase();
62
+
48
63
foreach ($patch_lists as $patch_list) {
49
-
$last_key = null;
64
+
$last_keys = array_fill_keys(
65
+
array_keys($phases),
66
+
null);
67
+
50
68
foreach ($patch_list->getPatches() as $key => $patch) {
51
69
if (!is_array($patch)) {
52
70
throw new Exception(
···
63
81
'after' => true,
64
82
'legacy' => true,
65
83
'dead' => true,
84
+
'phase' => true,
66
85
);
67
86
68
87
foreach ($patch as $pkey => $pval) {
···
128
147
$patch['legacy'] = false;
129
148
}
130
149
150
+
if (!array_key_exists('phase', $patch)) {
151
+
$patch['phase'] = $default_phase;
152
+
}
153
+
154
+
$patch_phase = $patch['phase'];
155
+
156
+
if (!isset($phases[$patch_phase])) {
157
+
throw new Exception(
158
+
pht(
159
+
'Storage patch "%s" specifies it should apply in phase "%s", '.
160
+
'but this phase is unrecognized. Valid phases are: %s.',
161
+
$full_key,
162
+
$patch_phase,
163
+
implode(', ', array_keys($phases))));
164
+
}
165
+
166
+
$last_key = $last_keys[$patch_phase];
167
+
131
168
if (!array_key_exists('after', $patch)) {
132
-
if ($last_key === null) {
169
+
if ($last_key === null && $patch_phase === $default_phase) {
133
170
throw new Exception(
134
171
pht(
135
172
"Patch '%s' is missing key 'after', and is the first patch ".
···
139
176
$full_key,
140
177
get_class($patch_list)));
141
178
} else {
142
-
$patch['after'] = array($last_key);
179
+
if ($last_key === null) {
180
+
$patch['after'] = array();
181
+
} else {
182
+
$patch['after'] = array($last_key);
183
+
}
143
184
}
144
185
}
145
-
$last_key = $full_key;
186
+
$last_keys[$patch_phase] = $full_key;
146
187
147
188
foreach ($patch['after'] as $after_key => $after) {
148
189
if (strpos($after, ':') === false) {
···
186
227
$key,
187
228
$after));
188
229
}
230
+
231
+
$patch_phase = $patch['phase'];
232
+
$after_phase = $specs[$after]['phase'];
233
+
234
+
if ($patch_phase !== $after_phase) {
235
+
throw new Exception(
236
+
pht(
237
+
'Storage patch "%s" executes in phase "%s", but depends on '.
238
+
'patch "%s" which is in a different phase ("%s"). Patches '.
239
+
'may not have dependencies across phases.',
240
+
$key,
241
+
$patch_phase,
242
+
$after,
243
+
$after_phase));
244
+
}
189
245
}
190
246
}
191
247
···
196
252
197
253
// TODO: Detect cycles?
198
254
255
+
$patches = msortv($patches, 'newSortVector');
256
+
199
257
return $patches;
258
+
}
259
+
260
+
private function getPHPPatchAttributes($patch_name, $full_path) {
261
+
$data = Filesystem::readFile($full_path);
262
+
263
+
$phase_list = PhabricatorStoragePatch::getPhaseList();
264
+
$phase_map = array_fuse($phase_list);
265
+
266
+
$attributes = array();
267
+
268
+
$lines = phutil_split_lines($data, false);
269
+
foreach ($lines as $line) {
270
+
// Skip over the "PHP" line.
271
+
if (preg_match('(^<\?)', $line)) {
272
+
continue;
273
+
}
274
+
275
+
// Skip over blank lines.
276
+
if (!strlen(trim($line))) {
277
+
continue;
278
+
}
279
+
280
+
// If this is a "//" comment...
281
+
if (preg_match('(^\s*//)', $line)) {
282
+
$matches = null;
283
+
if (preg_match('(^\s*//\s*@(\S+)(?:\s+(.*))?\z)', $line, $matches)) {
284
+
$attr_key = $matches[1];
285
+
$attr_value = trim(idx($matches, 2));
286
+
287
+
switch ($attr_key) {
288
+
case 'phase':
289
+
$phase_name = $attr_value;
290
+
291
+
if (!strlen($phase_name)) {
292
+
throw new Exception(
293
+
pht(
294
+
'Storage patch "%s" specifies a "@phase" attribute with '.
295
+
'no phase value. Phase attributes must specify a value, '.
296
+
'like "@phase default".',
297
+
$patch_name));
298
+
}
299
+
300
+
if (!isset($phase_map[$phase_name])) {
301
+
throw new Exception(
302
+
pht(
303
+
'Storage patch "%s" specifies a "@phase" value ("%s"), '.
304
+
'but this is not a recognized phase. Valid phases '.
305
+
'are: %s.',
306
+
$patch_name,
307
+
$phase_name,
308
+
implode(', ', $phase_list)));
309
+
}
310
+
311
+
if (isset($attributes['phase'])) {
312
+
throw new Exception(
313
+
pht(
314
+
'Storage patch "%s" specifies a "@phase" value ("%s"), '.
315
+
'but it already has a specified phase ("%s"). Patches '.
316
+
'may not specify multiple phases.',
317
+
$patch_name,
318
+
$phase_name,
319
+
$attributes['phase']));
320
+
}
321
+
322
+
$attributes[$attr_key] = $phase_name;
323
+
break;
324
+
default:
325
+
throw new Exception(
326
+
pht(
327
+
'Storage patch "%s" specifies attribute "%s", but this '.
328
+
'attribute is unknown.',
329
+
$patch_name,
330
+
$attr_key));
331
+
}
332
+
}
333
+
continue;
334
+
}
335
+
336
+
// If this is anything else, we're all done. Attributes must be marked
337
+
// in the header of the file.
338
+
break;
339
+
}
340
+
341
+
342
+
return $attributes;
200
343
}
201
344
202
345
}