@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Make daemons ignore "Unreachable" commits and avoid duplicate work
Summary:
Ref T9028. This improves the daemon behavior for unreachable commits. There is still no way for commits to become marked unreachable on their own.
- When a daemon encounters an unreachable commit, fail permanently.
- When we revive a commit, queue new daemons to process it (since some of the daemons might have failed permanently the first time around).
- Before doing a step on a commit, check if the step has already been done and skip it if it has. This can't happen normally, but will soon be possible if a commit is repeatedly deleted and revived very quickly.
- Steps queued with `bin/repository reparse ...` still execute normally.
Test Plan:
- Used `bin/repository reparse` to run every step, verified they all mark the commit with the proper flag.
- Faked the `reparse` exception in the "skip step" code, used `repository reparse` to skip every step.
- Marked a commit as unreachable, ran `discover`, saw daemons queue for it.
- Ran daemons with `bin/worker execute --id ...`, saw them all skip + queue the next step.
- Marked a commit as unreachable, ran `bin/repository reparse` on it, got permanent failures immediately for each step.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T9028
Differential Revision: https://secure.phabricator.com/D16131
+148
-97
+31
-16
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
+31
-16
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
···
514
514
$repository->getID(),
515
515
$commit_identifier);
516
516
if ($conn_w->getAffectedRows()) {
517
+
$commit = $commit->loadOneWhere(
518
+
'repositoryID = %d AND commitIdentifier = %s',
519
+
$repository->getID(),
520
+
$commit_identifier);
521
+
522
+
// After reviving a commit, schedule new daemons for it.
523
+
$this->didDiscoverCommit($repository, $commit, $epoch);
517
524
return;
518
525
}
519
-
520
526
521
527
$commit->setRepositoryID($repository->getID());
522
528
$commit->setCommitIdentifier($commit_identifier);
···
575
581
}
576
582
$commit->saveTransaction();
577
583
578
-
$this->insertTask($repository, $commit);
579
-
580
-
queryfx(
581
-
$conn_w,
582
-
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
583
-
VALUES (%d, 1, %d, %d)
584
-
ON DUPLICATE KEY UPDATE
585
-
size = size + 1,
586
-
lastCommitID =
587
-
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
588
-
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
589
-
PhabricatorRepository::TABLE_SUMMARY,
590
-
$repository->getID(),
591
-
$commit->getID(),
592
-
$epoch);
584
+
$this->didDiscoverCommit($repository, $commit, $epoch);
593
585
594
586
if ($this->repairMode) {
595
587
// Normally, the query should throw a duplicate key exception. If we
···
612
604
// data inconsistency or cosmic radiation; in any case, we're still
613
605
// in a good state if we ignore the failure.
614
606
}
607
+
}
608
+
609
+
private function didDiscoverCommit(
610
+
PhabricatorRepository $repository,
611
+
PhabricatorRepositoryCommit $commit,
612
+
$epoch) {
613
+
614
+
$this->insertTask($repository, $commit);
615
+
616
+
// Update the repository summary table.
617
+
queryfx(
618
+
$commit->establishConnection('w'),
619
+
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
620
+
VALUES (%d, 1, %d, %d)
621
+
ON DUPLICATE KEY UPDATE
622
+
size = size + 1,
623
+
lastCommitID =
624
+
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
625
+
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
626
+
PhabricatorRepository::TABLE_SUMMARY,
627
+
$repository->getID(),
628
+
$commit->getID(),
629
+
$epoch);
615
630
}
616
631
617
632
private function didDiscoverRefs(array $refs) {
+10
src/applications/repository/worker/PhabricatorRepositoryCommitHeraldWorker.php
+10
src/applications/repository/worker/PhabricatorRepositoryCommitHeraldWorker.php
···
3
3
final class PhabricatorRepositoryCommitHeraldWorker
4
4
extends PhabricatorRepositoryCommitParserWorker {
5
5
6
+
protected function getImportStepFlag() {
7
+
return PhabricatorRepositoryCommit::IMPORTED_HERALD;
8
+
}
9
+
6
10
public function getRequiredLeaseTime() {
7
11
// Herald rules may take a long time to process.
8
12
return phutil_units('4 hours in seconds');
···
11
15
protected function parseCommit(
12
16
PhabricatorRepository $repository,
13
17
PhabricatorRepositoryCommit $commit) {
18
+
19
+
if ($this->shouldSkipImportStep()) {
20
+
// This worker has no followup tasks, so we can just bail out
21
+
// right away without queueing anything.
22
+
return;
23
+
}
14
24
15
25
// Reload the commit to pull commit data and audit requests.
16
26
$commit = id(new DiffusionCommitQuery())
+8
-4
src/applications/repository/worker/PhabricatorRepositoryCommitOwnersWorker.php
+8
-4
src/applications/repository/worker/PhabricatorRepositoryCommitOwnersWorker.php
···
3
3
final class PhabricatorRepositoryCommitOwnersWorker
4
4
extends PhabricatorRepositoryCommitParserWorker {
5
5
6
+
protected function getImportStepFlag() {
7
+
return PhabricatorRepositoryCommit::IMPORTED_OWNERS;
8
+
}
9
+
6
10
protected function parseCommit(
7
11
PhabricatorRepository $repository,
8
12
PhabricatorRepositoryCommit $commit) {
9
13
10
-
$this->triggerOwnerAudits($repository, $commit);
11
-
12
-
$commit->writeImportStatusFlag(
13
-
PhabricatorRepositoryCommit::IMPORTED_OWNERS);
14
+
if (!$this->shouldSkipImportStep()) {
15
+
$this->triggerOwnerAudits($repository, $commit);
16
+
$commit->writeImportStatusFlag($this->getImportStepFlag());
17
+
}
14
18
15
19
if ($this->shouldQueueFollowupTasks()) {
16
20
$this->queueTask(
+44
src/applications/repository/worker/PhabricatorRepositoryCommitParserWorker.php
+44
src/applications/repository/worker/PhabricatorRepositoryCommitParserWorker.php
···
26
26
pht('Commit "%s" does not exist.', $commit_id));
27
27
}
28
28
29
+
if ($commit->isUnreachable()) {
30
+
throw new PhabricatorWorkerPermanentFailureException(
31
+
pht(
32
+
'Commit "%s" has been deleted: it is no longer reachable from '.
33
+
'any ref.',
34
+
$commit_id));
35
+
}
36
+
29
37
$this->commit = $commit;
30
38
31
39
return $commit;
···
42
50
43
51
final protected function shouldQueueFollowupTasks() {
44
52
return !idx($this->getTaskData(), 'only');
53
+
}
54
+
55
+
protected function getImportStepFlag() {
56
+
return null;
57
+
}
58
+
59
+
final protected function shouldSkipImportStep() {
60
+
// If this step has already been performed and this is a "natural" task
61
+
// which was queued by the normal daemons, decline to do the work again.
62
+
// This mitigates races if commits are rapidly deleted and revived.
63
+
$flag = $this->getImportStepFlag();
64
+
if (!$flag) {
65
+
// This step doesn't have an associated flag.
66
+
return false;
67
+
}
68
+
69
+
$commit = $this->commit;
70
+
if (!$commit->isPartiallyImported($flag)) {
71
+
// This commit doesn't have the flag set yet.
72
+
return false;
73
+
}
74
+
75
+
76
+
if (!$this->shouldQueueFollowupTasks()) {
77
+
// This task was queued by administrative tools, so do the work even
78
+
// if it duplicates existing work.
79
+
return false;
80
+
}
81
+
82
+
$this->log(
83
+
"%s\n",
84
+
pht(
85
+
'Skipping import step; this step was previously completed for '.
86
+
'this commit.'));
87
+
88
+
return true;
45
89
}
46
90
47
91
abstract protected function parseCommit(
+13
-9
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
+13
-9
src/applications/repository/worker/commitchangeparser/PhabricatorRepositoryCommitChangeParserWorker.php
···
3
3
abstract class PhabricatorRepositoryCommitChangeParserWorker
4
4
extends PhabricatorRepositoryCommitParserWorker {
5
5
6
+
protected function getImportStepFlag() {
7
+
return PhabricatorRepositoryCommit::IMPORTED_CHANGE;
8
+
}
9
+
6
10
public function getRequiredLeaseTime() {
7
11
// It can take a very long time to parse commits; some commits in the
8
12
// Facebook repository affect many millions of paths. Acquire 24h leases.
···
23
27
return;
24
28
}
25
29
26
-
$results = $this->parseCommitChanges($repository, $commit);
27
-
if ($results) {
28
-
$this->writeCommitChanges($repository, $commit, $results);
30
+
if (!$this->shouldSkipImportStep()) {
31
+
$results = $this->parseCommitChanges($repository, $commit);
32
+
if ($results) {
33
+
$this->writeCommitChanges($repository, $commit, $results);
34
+
}
35
+
36
+
$commit->writeImportStatusFlag($this->getImportStepFlag());
37
+
38
+
PhabricatorSearchWorker::queueDocumentForIndexing($commit->getPHID());
29
39
}
30
40
31
41
$this->finishParse();
···
85
95
86
96
protected function finishParse() {
87
97
$commit = $this->commit;
88
-
89
-
$commit->writeImportStatusFlag(
90
-
PhabricatorRepositoryCommit::IMPORTED_CHANGE);
91
-
92
-
PhabricatorSearchWorker::queueDocumentForIndexing($commit->getPHID());
93
-
94
98
if ($this->shouldQueueFollowupTasks()) {
95
99
$this->queueTask(
96
100
'PhabricatorRepositoryCommitOwnersWorker',
+36
-26
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php
+36
-26
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php
···
3
3
abstract class PhabricatorRepositoryCommitMessageParserWorker
4
4
extends PhabricatorRepositoryCommitParserWorker {
5
5
6
-
abstract protected function parseCommitWithRef(
7
-
PhabricatorRepository $repository,
8
-
PhabricatorRepositoryCommit $commit,
9
-
DiffusionCommitRef $ref);
6
+
protected function getImportStepFlag() {
7
+
return PhabricatorRepositoryCommit::IMPORTED_MESSAGE;
8
+
}
9
+
10
+
abstract protected function getFollowupTaskClass();
10
11
11
12
final protected function parseCommit(
12
13
PhabricatorRepository $repository,
13
14
PhabricatorRepositoryCommit $commit) {
14
15
15
-
$viewer = PhabricatorUser::getOmnipotentUser();
16
+
if (!$this->shouldSkipImportStep()) {
17
+
$viewer = PhabricatorUser::getOmnipotentUser();
16
18
17
-
$refs_raw = DiffusionQuery::callConduitWithDiffusionRequest(
18
-
$viewer,
19
-
DiffusionRequest::newFromDictionary(
19
+
$refs_raw = DiffusionQuery::callConduitWithDiffusionRequest(
20
+
$viewer,
21
+
DiffusionRequest::newFromDictionary(
22
+
array(
23
+
'repository' => $repository,
24
+
'user' => $viewer,
25
+
)),
26
+
'diffusion.querycommits',
20
27
array(
21
-
'repository' => $repository,
22
-
'user' => $viewer,
23
-
)),
24
-
'diffusion.querycommits',
25
-
array(
26
-
'repositoryPHID' => $repository->getPHID(),
27
-
'phids' => array($commit->getPHID()),
28
-
'bypassCache' => true,
29
-
'needMessages' => true,
30
-
));
28
+
'repositoryPHID' => $repository->getPHID(),
29
+
'phids' => array($commit->getPHID()),
30
+
'bypassCache' => true,
31
+
'needMessages' => true,
32
+
));
31
33
32
-
if (empty($refs_raw['data'])) {
33
-
throw new Exception(
34
-
pht(
35
-
'Unable to retrieve details for commit "%s"!',
36
-
$commit->getPHID()));
34
+
if (empty($refs_raw['data'])) {
35
+
throw new Exception(
36
+
pht(
37
+
'Unable to retrieve details for commit "%s"!',
38
+
$commit->getPHID()));
39
+
}
40
+
41
+
$ref = DiffusionCommitRef::newFromConduitResult(head($refs_raw['data']));
42
+
$this->updateCommitData($ref);
37
43
}
38
44
39
-
$ref = DiffusionCommitRef::newFromConduitResult(head($refs_raw['data']));
40
-
41
-
$this->parseCommitWithRef($repository, $commit, $ref);
45
+
if ($this->shouldQueueFollowupTasks()) {
46
+
$this->queueTask(
47
+
$this->getFollowupTaskClass(),
48
+
array(
49
+
'commitID' => $commit->getID(),
50
+
));
51
+
}
42
52
}
43
53
44
54
final protected function updateCommitData(DiffusionCommitRef $ref) {
+2
-14
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryGitCommitMessageParserWorker.php
+2
-14
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryGitCommitMessageParserWorker.php
···
3
3
final class PhabricatorRepositoryGitCommitMessageParserWorker
4
4
extends PhabricatorRepositoryCommitMessageParserWorker {
5
5
6
-
protected function parseCommitWithRef(
7
-
PhabricatorRepository $repository,
8
-
PhabricatorRepositoryCommit $commit,
9
-
DiffusionCommitRef $ref) {
10
-
11
-
$this->updateCommitData($ref);
12
-
13
-
if ($this->shouldQueueFollowupTasks()) {
14
-
$this->queueTask(
15
-
'PhabricatorRepositoryGitCommitChangeParserWorker',
16
-
array(
17
-
'commitID' => $commit->getID(),
18
-
));
19
-
}
6
+
protected function getFollowupTaskClass() {
7
+
return 'PhabricatorRepositoryGitCommitChangeParserWorker';
20
8
}
21
9
22
10
}
+2
-14
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php
+2
-14
src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryMercurialCommitMessageParserWorker.php
···
3
3
final class PhabricatorRepositoryMercurialCommitMessageParserWorker
4
4
extends PhabricatorRepositoryCommitMessageParserWorker {
5
5
6
-
protected function parseCommitWithRef(
7
-
PhabricatorRepository $repository,
8
-
PhabricatorRepositoryCommit $commit,
9
-
DiffusionCommitRef $ref) {
10
-
11
-
$this->updateCommitData($ref);
12
-
13
-
if ($this->shouldQueueFollowupTasks()) {
14
-
$this->queueTask(
15
-
'PhabricatorRepositoryMercurialCommitChangeParserWorker',
16
-
array(
17
-
'commitID' => $commit->getID(),
18
-
));
19
-
}
6
+
protected function getFollowupTaskClass() {
7
+
return 'PhabricatorRepositoryMercurialCommitChangeParserWorker';
20
8
}
21
9
22
10
}
+2
-14
src/applications/repository/worker/commitmessageparser/PhabricatorRepositorySvnCommitMessageParserWorker.php
+2
-14
src/applications/repository/worker/commitmessageparser/PhabricatorRepositorySvnCommitMessageParserWorker.php
···
3
3
final class PhabricatorRepositorySvnCommitMessageParserWorker
4
4
extends PhabricatorRepositoryCommitMessageParserWorker {
5
5
6
-
protected function parseCommitWithRef(
7
-
PhabricatorRepository $repository,
8
-
PhabricatorRepositoryCommit $commit,
9
-
DiffusionCommitRef $ref) {
10
-
11
-
$this->updateCommitData($ref);
12
-
13
-
if ($this->shouldQueueFollowupTasks()) {
14
-
$this->queueTask(
15
-
'PhabricatorRepositorySvnCommitChangeParserWorker',
16
-
array(
17
-
'commitID' => $commit->getID(),
18
-
));
19
-
}
6
+
protected function getFollowupTaskClass() {
7
+
return 'PhabricatorRepositorySvnCommitChangeParserWorker';
20
8
}
21
9
22
10
}