@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Update major RefCursor callsites to work properly with RefPosition
Summary:
Ref T11823. This is the meaty part of the change, and updates `RefEngine` to use separate RefCursor (for names) and RefPosition (for actual commit positions) tables.
I'll hold this whole series until after the release cut so it has some time to bake on `secure` to look for issues. It's also not a huge problem if there are bugs here since these tables are just caches anyway, although they do feed into some other things, and obviously it's never good to have bugs.
Test Plan:
- This logic can be invoked directly with `bin/repository refs <repository> --trace --verbose`.
- Ran that on unchanged repositories, new branches, removed branches, and modified branches. Saw appropriate output and cursor positions.
- Ran on a mercurial repository to test the close/open logic, saw it correct open/closed state of incorrect positions.
- Browed around Diffusion in various repositories.
Reviewers: amckinley
Reviewed By: amckinley
Maniphest Tasks: T11823
Differential Revision: https://secure.phabricator.com/D18614
+233
-86
+2
-12
src/applications/differential/controller/DifferentialRevisionOperationController.php
+2
-12
src/applications/differential/controller/DifferentialRevisionOperationController.php
···
137
137
return null;
138
138
}
139
139
140
-
// NOTE: See PHI68. This is a workaround to make "Land Revision" work
141
-
// until T11823 is fixed properly. If we find multiple refs with the same
142
-
// name (normally, duplicate "master" refs), just pick the first one.
143
-
144
-
$refs = $this->newRefQuery($repository)
140
+
return $this->newRefQuery($repository)
145
141
->withRefNames(array($default_name))
146
-
->execute();
147
-
148
-
if ($refs) {
149
-
return head($refs);
150
-
}
151
-
152
-
return null;
142
+
->executeOne();
153
143
}
154
144
155
145
private function getDefaultRefName(
+4
-2
src/applications/diffusion/query/DiffusionCachedResolveRefsQuery.php
+4
-2
src/applications/diffusion/query/DiffusionCachedResolveRefsQuery.php
···
106
106
107
107
$cursors = queryfx_all(
108
108
$conn_r,
109
-
'SELECT refNameHash, refType, commitIdentifier, isClosed FROM %T
110
-
WHERE repositoryPHID = %s AND refNameHash IN (%Ls)',
109
+
'SELECT c.refNameHash, c.refType, p.commitIdentifier, p.isClosed
110
+
FROM %T c JOIN %T p ON p.cursorID = c.id
111
+
WHERE c.repositoryPHID = %s AND c.refNameHash IN (%Ls)',
111
112
id(new PhabricatorRepositoryRefCursor())->getTableName(),
113
+
id(new PhabricatorRepositoryRefPosition())->getTableName(),
112
114
$repository->getPHID(),
113
115
array_keys($name_hashes));
114
116
+190
-72
src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
+190
-72
src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
···
7
7
final class PhabricatorRepositoryRefEngine
8
8
extends PhabricatorRepositoryEngine {
9
9
10
-
private $newRefs = array();
11
-
private $deadRefs = array();
10
+
private $newPositions = array();
11
+
private $deadPositions = array();
12
12
private $closeCommits = array();
13
13
private $hasNoCursors;
14
14
15
15
public function updateRefs() {
16
-
$this->newRefs = array();
17
-
$this->deadRefs = array();
16
+
$this->newPositions = array();
17
+
$this->deadPositions = array();
18
18
$this->closeCommits = array();
19
19
20
20
$repository = $this->getRepository();
21
+
$viewer = $this->getViewer();
21
22
22
23
$branches_may_close = false;
23
24
···
53
54
);
54
55
55
56
$all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
56
-
->setViewer(PhabricatorUser::getOmnipotentUser())
57
+
->setViewer($viewer)
57
58
->withRepositoryPHIDs(array($repository->getPHID()))
59
+
->needPositions(true)
58
60
->execute();
59
61
$cursor_groups = mgroup($all_cursors, 'getRefType');
60
62
···
63
65
// Find all the heads of closing refs.
64
66
$all_closing_heads = array();
65
67
foreach ($all_cursors as $cursor) {
66
-
if ($this->shouldCloseRef($cursor->getRefType(), $cursor->getRefName())) {
67
-
$all_closing_heads[] = $cursor->getCommitIdentifier();
68
+
$should_close = $this->shouldCloseRef(
69
+
$cursor->getRefType(),
70
+
$cursor->getRefName());
71
+
if (!$should_close) {
72
+
continue;
73
+
}
74
+
75
+
foreach ($cursor->getPositionIdentifiers() as $identifier) {
76
+
$all_closing_heads[] = $identifier;
68
77
}
69
78
}
70
79
$all_closing_heads = array_unique($all_closing_heads);
···
79
88
$this->setCloseFlagOnCommits($this->closeCommits);
80
89
}
81
90
82
-
if ($this->newRefs || $this->deadRefs) {
91
+
if ($this->newPositions || $this->deadPositions) {
83
92
$repository->openTransaction();
84
-
foreach ($this->newRefs as $ref) {
85
-
$ref->save();
86
-
}
87
-
foreach ($this->deadRefs as $ref) {
88
-
// Shove this ref into the old refs table so the discovery engine
89
-
// can check if any commits have been rendered unreachable.
90
-
id(new PhabricatorRepositoryOldRef())
91
-
->setRepositoryPHID($repository->getPHID())
92
-
->setCommitIdentifier($ref->getCommitIdentifier())
93
-
->save();
93
+
94
+
$this->saveNewPositions();
95
+
$this->deleteDeadPositions();
94
96
95
-
$ref->delete();
96
-
}
97
97
$repository->saveTransaction();
98
-
99
-
$this->newRefs = array();
100
-
$this->deadRefs = array();
101
98
}
102
99
103
100
$branches = $maps[PhabricatorRepositoryRefCursor::TYPE_BRANCH];
···
111
108
array $branches) {
112
109
113
110
assert_instances_of($branches, 'DiffusionRepositoryRef');
111
+
$viewer = $this->getViewer();
114
112
115
113
$all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
116
-
->setViewer(PhabricatorUser::getOmnipotentUser())
114
+
->setViewer($viewer)
117
115
->withRepositoryPHIDs(array($repository->getPHID()))
116
+
->needPositions(true)
118
117
->execute();
119
118
120
119
$state_map = array();
···
124
123
continue;
125
124
}
126
125
$raw_name = $cursor->getRefNameRaw();
127
-
$hash = $cursor->getCommitIdentifier();
128
126
129
-
$state_map[$raw_name][$hash] = $cursor;
127
+
foreach ($cursor->getPositions() as $position) {
128
+
$hash = $position->getCommitIdentifier();
129
+
$state_map[$raw_name][$hash] = $position;
130
+
}
130
131
}
131
132
133
+
$updates = array();
132
134
foreach ($branches as $branch) {
133
-
$cursor = idx($state_map, $branch->getShortName(), array());
134
-
$cursor = idx($cursor, $branch->getCommitIdentifier());
135
-
if (!$cursor) {
135
+
$position = idx($state_map, $branch->getShortName(), array());
136
+
$position = idx($position, $branch->getCommitIdentifier());
137
+
if (!$position) {
136
138
continue;
137
139
}
138
140
139
141
$fields = $branch->getRawFields();
140
142
141
-
$cursor_state = (bool)$cursor->getIsClosed();
143
+
$position_state = (bool)$position->getIsClosed();
142
144
$branch_state = (bool)idx($fields, 'closed');
143
145
144
-
if ($cursor_state != $branch_state) {
145
-
$cursor->setIsClosed((int)$branch_state)->save();
146
+
if ($position_state != $branch_state) {
147
+
$updates[$position->getID()] = (int)$branch_state;
146
148
}
147
149
}
150
+
151
+
if ($updates) {
152
+
$position_table = id(new PhabricatorRepositoryRefPosition());
153
+
$conn = $position_table->establishConnection('w');
154
+
155
+
$position_table->openTransaction();
156
+
foreach ($updates as $position_id => $branch_state) {
157
+
queryfx(
158
+
$conn,
159
+
'UPDATE %T SET isClosed = %d WHERE id = %d',
160
+
$position_table->getTableName(),
161
+
$branch_state,
162
+
$position_id);
163
+
}
164
+
$position_table->saveTransaction();
165
+
}
148
166
}
149
167
150
-
private function markRefNew(PhabricatorRepositoryRefCursor $cursor) {
151
-
$this->newRefs[] = $cursor;
168
+
private function markPositionNew(
169
+
PhabricatorRepositoryRefPosition $position) {
170
+
$this->newPositions[] = $position;
152
171
return $this;
153
172
}
154
173
155
-
private function markRefDead(PhabricatorRepositoryRefCursor $cursor) {
156
-
$this->deadRefs[] = $cursor;
174
+
private function markPositionDead(
175
+
PhabricatorRepositoryRefPosition $position) {
176
+
$this->deadPositions[] = $position;
157
177
return $this;
158
178
}
159
179
···
203
223
// NOTE: Mercurial branches may have multiple branch heads; this logic
204
224
// is complex primarily to account for that.
205
225
206
-
// Group all the cursors by their ref name, like "master". Since Mercurial
207
-
// branches may have multiple heads, there could be several cursors with
208
-
// the same name.
209
-
$cursor_groups = mgroup($cursors, 'getRefNameRaw');
226
+
$cursors = mpull($cursors, null, 'getRefNameRaw');
210
227
211
228
// Group all the new ref values by their name. As above, these groups may
212
229
// have multiple members in Mercurial.
···
215
232
foreach ($ref_groups as $name => $refs) {
216
233
$new_commits = mpull($refs, 'getCommitIdentifier', 'getCommitIdentifier');
217
234
218
-
$ref_cursors = idx($cursor_groups, $name, array());
219
-
$old_commits = mpull($ref_cursors, null, 'getCommitIdentifier');
235
+
$ref_cursor = idx($cursors, $name);
236
+
if ($ref_cursor) {
237
+
$old_positions = $ref_cursor->getPositions();
238
+
} else {
239
+
$old_positions = array();
240
+
}
220
241
221
242
// We're going to delete all the cursors pointing at commits which are
222
243
// no longer associated with the refs. This primarily makes the Mercurial
223
244
// multiple head case easier, and means that when we update a ref we
224
245
// delete the old one and write a new one.
225
-
foreach ($ref_cursors as $cursor) {
226
-
if (isset($new_commits[$cursor->getCommitIdentifier()])) {
246
+
foreach ($old_positions as $old_position) {
247
+
$hash = $old_position->getCommitIdentifier();
248
+
if (isset($new_commits[$hash])) {
227
249
// This ref previously pointed at this commit, and still does.
228
250
$this->log(
229
251
pht(
230
252
'Ref %s "%s" still points at %s.',
231
253
$ref_type,
232
254
$name,
233
-
$cursor->getCommitIdentifier()));
234
-
} else {
235
-
// This ref previously pointed at this commit, but no longer does.
236
-
$this->log(
237
-
pht(
238
-
'Ref %s "%s" no longer points at %s.',
239
-
$ref_type,
240
-
$name,
241
-
$cursor->getCommitIdentifier()));
242
-
243
-
// Nuke the obsolete cursor.
244
-
$this->markRefDead($cursor);
255
+
$hash));
256
+
continue;
245
257
}
258
+
259
+
// This ref previously pointed at this commit, but no longer does.
260
+
$this->log(
261
+
pht(
262
+
'Ref %s "%s" no longer points at %s.',
263
+
$ref_type,
264
+
$name,
265
+
$hash));
266
+
267
+
// Nuke the obsolete cursor.
268
+
$this->markPositionDead($old_position);
246
269
}
247
270
248
271
// Now, we're going to insert new cursors for all the commits which are
249
272
// associated with this ref that don't currently have cursors.
273
+
$old_commits = mpull($old_positions, 'getCommitIdentifier');
274
+
$old_commits = array_fuse($old_commits);
275
+
250
276
$added_commits = array_diff_key($new_commits, $old_commits);
251
277
foreach ($added_commits as $identifier) {
252
278
$this->log(
···
255
281
$ref_type,
256
282
$name,
257
283
$identifier));
258
-
$this->markRefNew(
259
-
id(new PhabricatorRepositoryRefCursor())
260
-
->setRepositoryPHID($repository->getPHID())
261
-
->setRefType($ref_type)
262
-
->setRefName($name)
263
-
->setCommitIdentifier($identifier));
284
+
285
+
if (!$ref_cursor) {
286
+
// If this is the first time we've seen a particular ref (for
287
+
// example, a new branch) we need to insert a RefCursor record
288
+
// for it before we can insert a RefPosition.
289
+
290
+
$ref_cursor = $this->newRefCursor(
291
+
$repository,
292
+
$ref_type,
293
+
$name);
294
+
}
295
+
296
+
$new_position = id(new PhabricatorRepositoryRefPosition())
297
+
->setCursorID($ref_cursor->getID())
298
+
->setCommitIdentifier($identifier)
299
+
->setIsClosed(0);
300
+
301
+
$this->markPositionNew($new_position);
264
302
}
265
303
266
304
if ($this->shouldCloseRef($ref_type, $name)) {
···
277
315
// Find any cursors for refs which no longer exist. This happens when a
278
316
// branch, tag or bookmark is deleted.
279
317
280
-
foreach ($cursor_groups as $name => $cursor_group) {
281
-
if (idx($ref_groups, $name) === null) {
282
-
foreach ($cursor_group as $cursor) {
283
-
$this->log(
284
-
pht(
285
-
'Ref %s "%s" no longer exists.',
286
-
$cursor->getRefType(),
287
-
$cursor->getRefName()));
288
-
$this->markRefDead($cursor);
289
-
}
318
+
foreach ($cursors as $name => $cursor) {
319
+
if (!empty($ref_groups[$name])) {
320
+
// This ref still has some positions, so we don't need to wipe it
321
+
// out. Try the next one.
322
+
continue;
323
+
}
324
+
325
+
foreach ($cursor->getPositions() as $position) {
326
+
$this->log(
327
+
pht(
328
+
'Ref %s "%s" no longer exists.',
329
+
$cursor->getRefType(),
330
+
$cursor->getRefName()));
331
+
332
+
$this->markPositionDead($position);
290
333
}
291
334
}
292
335
}
···
451
494
452
495
return $this;
453
496
}
497
+
498
+
private function newRefCursor(
499
+
PhabricatorRepository $repository,
500
+
$ref_type,
501
+
$ref_name) {
502
+
503
+
$cursor = id(new PhabricatorRepositoryRefCursor())
504
+
->setRepositoryPHID($repository->getPHID())
505
+
->setRefType($ref_type)
506
+
->setRefName($ref_name);
507
+
508
+
try {
509
+
return $cursor->save();
510
+
} catch (AphrontDuplicateKeyQueryException $ex) {
511
+
// If we raced another daemon to create this position and lost the race,
512
+
// load the cursor the other daemon created instead.
513
+
}
514
+
515
+
$viewer = $this->getViewer();
516
+
517
+
$cursor = id(new PhabricatorRepositoryRefCursorQuery())
518
+
->setViewer($viewer)
519
+
->withRepositoryPHIDs(array($repository->getPHID()))
520
+
->withRefTypes(array($ref_type))
521
+
->withRefNames(array($ref_name))
522
+
->needPositions(true)
523
+
->executeOne();
524
+
if (!$cursor) {
525
+
throw new Exception(
526
+
pht(
527
+
'Failed to create a new ref cursor (for "%s", of type "%s", in '.
528
+
'repository "%s") because it collided with an existing cursor, '.
529
+
'but then failed to load that cursor.',
530
+
$ref_name,
531
+
$ref_type,
532
+
$repository->getDisplayName()));
533
+
}
534
+
535
+
return $cursor;
536
+
}
537
+
538
+
private function saveNewPositions() {
539
+
$positions = $this->newPositions;
540
+
541
+
foreach ($positions as $position) {
542
+
try {
543
+
$position->save();
544
+
} catch (AphrontDuplicateKeyQueryException $ex) {
545
+
// We may race another daemon to create this position. If we do, and
546
+
// we lose the race, that's fine: the other daemon did our work for
547
+
// us and we can continue.
548
+
}
549
+
}
550
+
551
+
$this->newPositions = array();
552
+
}
553
+
554
+
private function deleteDeadPositions() {
555
+
$positions = $this->deadPositions;
556
+
$repository = $this->getRepository();
557
+
558
+
foreach ($positions as $position) {
559
+
// Shove this ref into the old refs table so the discovery engine
560
+
// can check if any commits have been rendered unreachable.
561
+
id(new PhabricatorRepositoryOldRef())
562
+
->setRepositoryPHID($repository->getPHID())
563
+
->setCommitIdentifier($position->getCommitIdentifier())
564
+
->save();
565
+
566
+
$position->delete();
567
+
}
568
+
569
+
$this->deadPositions = array();
570
+
}
571
+
454
572
455
573
456
574
/* -( Updating Git Refs )-------------------------------------------------- */
+22
src/applications/repository/query/PhabricatorRepositoryRefCursorQuery.php
+22
src/applications/repository/query/PhabricatorRepositoryRefCursorQuery.php
···
9
9
private $refTypes;
10
10
private $refNames;
11
11
private $datasourceQuery;
12
+
private $needPositions;
12
13
13
14
public function withIDs(array $ids) {
14
15
$this->ids = $ids;
···
37
38
38
39
public function withDatasourceQuery($query) {
39
40
$this->datasourceQuery = $query;
41
+
return $this;
42
+
}
43
+
44
+
public function needPositions($need) {
45
+
$this->needPositions = $need;
40
46
return $this;
41
47
}
42
48
···
66
72
continue;
67
73
}
68
74
$ref->attachRepository($repository);
75
+
}
76
+
77
+
if (!$refs) {
78
+
return $refs;
79
+
}
80
+
81
+
if ($this->needPositions) {
82
+
$positions = id(new PhabricatorRepositoryRefPosition())->loadAllWhere(
83
+
'cursorID IN (%Ld)',
84
+
mpull($refs, 'getID'));
85
+
$positions = mgroup($positions, 'getCursorID');
86
+
87
+
foreach ($refs as $key => $ref) {
88
+
$ref_positions = idx($positions, $ref->getID(), array());
89
+
$ref->attachPositions($ref_positions);
90
+
}
69
91
}
70
92
71
93
return $refs;
+15
src/applications/repository/storage/PhabricatorRepositoryRefCursor.php
+15
src/applications/repository/storage/PhabricatorRepositoryRefCursor.php
···
21
21
protected $refNameEncoding;
22
22
23
23
private $repository = self::ATTACHABLE;
24
+
private $positions = self::ATTACHABLE;
24
25
25
26
protected function getConfiguration() {
26
27
return array(
···
69
70
70
71
public function getRepository() {
71
72
return $this->assertAttached($this->repository);
73
+
}
74
+
75
+
public function attachPositions(array $positions) {
76
+
assert_instances_of($positions, 'PhabricatorRepositoryRefPosition');
77
+
$this->positions = $positions;
78
+
return $this;
79
+
}
80
+
81
+
public function getPositions() {
82
+
return $this->assertAttached($this->positions);
83
+
}
84
+
85
+
public function getPositionIdentifiers() {
86
+
return mpull($this->getPositions(), 'getCommitIdentifier');
72
87
}
73
88
74
89