@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Lift logic for queueing commit import tasks into RepositoryEngine
Summary:
Ref T13591. There are currently two pathways to queue an import task for a commit: via repository discovery, or via a ref becoming permanent.
These pathways duplicate some logic and have behavioral differences: one does not set `objectPHID` properly, one does not set the priority correctly.
Unify these pathways, make them both set `objectPHID`, and make them both use the same priority logic.
Test Plan:
- Discovered refs.
- See later changes in this series for more complete test cases.
Maniphest Tasks: T13591
Differential Revision: https://secure.phabricator.com/D21516
+138
-109
+6
-98
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
+6
-98
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
···
655
655
$epoch,
656
656
$task_priority) {
657
657
658
-
$this->insertTask($repository, $commit, $task_priority);
658
+
$this->queueCommitImportTask(
659
+
$repository,
660
+
$commit->getID(),
661
+
$commit->getPHID(),
662
+
$task_priority,
663
+
$via = 'discovery');
659
664
660
665
// Update the repository summary table.
661
666
queryfx(
···
677
682
foreach ($refs as $ref) {
678
683
$this->workingSet[$ref->getIdentifier()] = true;
679
684
}
680
-
}
681
-
682
-
private function insertTask(
683
-
PhabricatorRepository $repository,
684
-
PhabricatorRepositoryCommit $commit,
685
-
$task_priority,
686
-
$data = array()) {
687
-
688
-
$vcs = $repository->getVersionControlSystem();
689
-
switch ($vcs) {
690
-
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
691
-
$class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
692
-
break;
693
-
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
694
-
$class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
695
-
break;
696
-
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
697
-
$class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
698
-
break;
699
-
default:
700
-
throw new Exception(pht("Unknown repository type '%s'!", $vcs));
701
-
}
702
-
703
-
$data['commitID'] = $commit->getID();
704
-
705
-
$options = array(
706
-
'priority' => $task_priority,
707
-
);
708
-
709
-
PhabricatorWorker::scheduleTask($class, $data, $options);
710
685
}
711
686
712
687
private function isInitialImport(array $refs) {
···
924
899
$data['N'],
925
900
$data['id'],
926
901
$data['epoch']);
927
-
}
928
-
929
-
private function getImportTaskPriority(
930
-
PhabricatorRepository $repository,
931
-
array $refs) {
932
-
933
-
// If the repository is importing for the first time, we schedule tasks
934
-
// at IMPORT priority, which is very low. Making progress on importing a
935
-
// new repository for the first time is less important than any other
936
-
// daemon task.
937
-
938
-
// If the repository has finished importing and we're just catching up
939
-
// on recent commits, we usually schedule discovery at COMMIT priority,
940
-
// which is slightly below the default priority.
941
-
942
-
// Note that followup tasks and triggered tasks (like those generated by
943
-
// Herald or Harbormaster) will queue at DEFAULT priority, so that each
944
-
// commit tends to fully import before we start the next one. This tends
945
-
// to give imports fairly predictable progress. See T11677 for some
946
-
// discussion.
947
-
948
-
if ($repository->isImporting()) {
949
-
$this->log(
950
-
pht(
951
-
'Importing %s commit(s) at low priority ("PRIORITY_IMPORT") '.
952
-
'because this repository is still importing.',
953
-
phutil_count($refs)));
954
-
955
-
return PhabricatorWorker::PRIORITY_IMPORT;
956
-
}
957
-
958
-
// See T13369. If we've discovered a lot of commits at once, import them
959
-
// at lower priority.
960
-
961
-
// This is mostly aimed at reducing the impact that synchronizing thousands
962
-
// of commits from a remote upstream has on other repositories. The queue
963
-
// is "mostly FIFO", so queueing a thousand commit imports can stall other
964
-
// repositories.
965
-
966
-
// In a perfect world we'd probably give repositories round-robin queue
967
-
// priority, but we don't currently have the primitives for this and there
968
-
// isn't a strong case for building them.
969
-
970
-
// Use "a whole lot of commits showed up at once" as a heuristic for
971
-
// detecting "someone synchronized an upstream", and import them at a lower
972
-
// priority to more closely approximate fair scheduling.
973
-
974
-
if (count($refs) >= PhabricatorRepository::LOWPRI_THRESHOLD) {
975
-
$this->log(
976
-
pht(
977
-
'Importing %s commit(s) at low priority ("PRIORITY_IMPORT") '.
978
-
'because many commits were discovered at once.',
979
-
phutil_count($refs)));
980
-
981
-
return PhabricatorWorker::PRIORITY_IMPORT;
982
-
}
983
-
984
-
// Otherwise, import at normal priority.
985
-
986
-
if ($refs) {
987
-
$this->log(
988
-
pht(
989
-
'Importing %s commit(s) at normal priority ("PRIORITY_COMMIT").',
990
-
phutil_count($refs)));
991
-
}
992
-
993
-
return PhabricatorWorker::PRIORITY_COMMIT;
994
902
}
995
903
996
904
}
+110
src/applications/repository/engine/PhabricatorRepositoryEngine.php
+110
src/applications/repository/engine/PhabricatorRepositoryEngine.php
···
83
83
return $this;
84
84
}
85
85
86
+
final protected function queueCommitImportTask(
87
+
PhabricatorRepository $repository,
88
+
$commit_id,
89
+
$commit_phid,
90
+
$task_priority,
91
+
$via = null) {
92
+
93
+
$vcs = $repository->getVersionControlSystem();
94
+
switch ($vcs) {
95
+
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
96
+
$class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
97
+
break;
98
+
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
99
+
$class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
100
+
break;
101
+
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
102
+
$class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
103
+
break;
104
+
default:
105
+
throw new Exception(
106
+
pht(
107
+
'Unknown repository type "%s"!',
108
+
$vcs));
109
+
}
110
+
111
+
$data = array(
112
+
'commitID' => $commit_id,
113
+
);
114
+
115
+
if ($via !== null) {
116
+
$data['via'] = $via;
117
+
}
118
+
119
+
$options = array(
120
+
'priority' => $task_priority,
121
+
'objectPHID' => $commit_phid,
122
+
);
123
+
124
+
PhabricatorWorker::scheduleTask($class, $data, $options);
125
+
}
126
+
127
+
final protected function getImportTaskPriority(
128
+
PhabricatorRepository $repository,
129
+
array $refs) {
130
+
assert_instances_of($refs, 'PhabricatorRepositoryCommitRef');
131
+
132
+
// If the repository is importing for the first time, we schedule tasks
133
+
// at IMPORT priority, which is very low. Making progress on importing a
134
+
// new repository for the first time is less important than any other
135
+
// daemon task.
136
+
137
+
// If the repository has finished importing and we're just catching up
138
+
// on recent commits, we usually schedule discovery at COMMIT priority,
139
+
// which is slightly below the default priority.
140
+
141
+
// Note that followup tasks and triggered tasks (like those generated by
142
+
// Herald or Harbormaster) will queue at DEFAULT priority, so that each
143
+
// commit tends to fully import before we start the next one. This tends
144
+
// to give imports fairly predictable progress. See T11677 for some
145
+
// discussion.
146
+
147
+
if ($repository->isImporting()) {
148
+
$this->log(
149
+
pht(
150
+
'Importing %s commit(s) at low priority ("PRIORITY_IMPORT") '.
151
+
'because this repository is still importing.',
152
+
phutil_count($refs)));
153
+
154
+
return PhabricatorWorker::PRIORITY_IMPORT;
155
+
}
156
+
157
+
// See T13369. If we've discovered a lot of commits at once, import them
158
+
// at lower priority.
159
+
160
+
// This is mostly aimed at reducing the impact that synchronizing thousands
161
+
// of commits from a remote upstream has on other repositories. The queue
162
+
// is "mostly FIFO", so queueing a thousand commit imports can stall other
163
+
// repositories.
164
+
165
+
// In a perfect world we'd probably give repositories round-robin queue
166
+
// priority, but we don't currently have the primitives for this and there
167
+
// isn't a strong case for building them.
168
+
169
+
// Use "a whole lot of commits showed up at once" as a heuristic for
170
+
// detecting "someone synchronized an upstream", and import them at a lower
171
+
// priority to more closely approximate fair scheduling.
172
+
173
+
if (count($refs) >= PhabricatorRepository::LOWPRI_THRESHOLD) {
174
+
$this->log(
175
+
pht(
176
+
'Importing %s commit(s) at low priority ("PRIORITY_IMPORT") '.
177
+
'because many commits were discovered at once.',
178
+
phutil_count($refs)));
179
+
180
+
return PhabricatorWorker::PRIORITY_IMPORT;
181
+
}
182
+
183
+
// Otherwise, import at normal priority.
184
+
185
+
if ($refs) {
186
+
$this->log(
187
+
pht(
188
+
'Importing %s commit(s) at normal priority ("PRIORITY_COMMIT").',
189
+
phutil_count($refs)));
190
+
}
191
+
192
+
return PhabricatorWorker::PRIORITY_COMMIT;
193
+
}
194
+
195
+
86
196
}
+22
-11
src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
+22
-11
src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
···
548
548
}
549
549
}
550
550
551
+
$commit_refs = array();
552
+
foreach ($identifiers as $identifier) {
553
+
554
+
// See T13591. This construction is a bit ad-hoc, but the priority
555
+
// function currently only cares about the number of refs we have
556
+
// discovered, so we'll get the right result even without filling
557
+
// these records out in detail.
558
+
559
+
$commit_refs[] = id(new PhabricatorRepositoryCommitRef())
560
+
->setIdentifier($identifier);
561
+
}
562
+
563
+
$task_priority = $this->getImportTaskPriority(
564
+
$repository,
565
+
$commit_refs);
566
+
551
567
$permanent_flag = PhabricatorRepositoryCommit::IMPORTED_PERMANENT;
552
568
$published_flag = PhabricatorRepositoryCommit::IMPORTED_PUBLISH;
553
569
···
580
596
$import_status,
581
597
$row['id']);
582
598
583
-
$data = array(
584
-
'commitID' => $row['id'],
585
-
);
586
-
587
-
PhabricatorWorker::scheduleTask(
588
-
$class,
589
-
$data,
590
-
array(
591
-
'priority' => PhabricatorWorker::PRIORITY_COMMIT,
592
-
'objectPHID' => $row['phid'],
593
-
));
599
+
$this->queueCommitImportTask(
600
+
$repository,
601
+
$row['id'],
602
+
$row['phid'],
603
+
$task_priority,
604
+
$via = 'ref');
594
605
}
595
606
}
596
607