@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
When many commits are discovered at once, import them at lower priority
Summary:
Ref T13369. See that task for discussion.
When the discovery daemon finds more than 64 commits to import, demote the worker queue priority of the resulting tasks.
Test Plan:
- Pushed one commit, ran `bin/repository discover --verbose --trace ...`, saw commit import with "at normal priority" message and priority 2500 ("PRIORITY_COMMIT").
- Pushed 3 commits, set threshold to 3, ran `bin/repository discover ...`, saw commist import with "at lower priority" message and priority 4000 ("PRIORITY_IMPORT").
Maniphest Tasks: T13369
Differential Revision: https://secure.phabricator.com/D20712
+81
-27
+79
-27
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
+79
-27
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
···
93
93
// Clear the working set cache.
94
94
$this->workingSet = array();
95
95
96
+
$task_priority = $this->getImportTaskPriority($repository, $refs);
97
+
96
98
// Record discovered commits and mark them in the cache.
97
99
foreach ($refs as $ref) {
98
100
$this->recordCommit(
···
100
102
$ref->getIdentifier(),
101
103
$ref->getEpoch(),
102
104
$ref->getCanCloseImmediately(),
103
-
$ref->getParents());
105
+
$ref->getParents(),
106
+
$task_priority);
104
107
105
108
$this->commitCache[$ref->getIdentifier()] = true;
106
109
}
···
536
539
$commit_identifier,
537
540
$epoch,
538
541
$close_immediately,
539
-
array $parents) {
542
+
array $parents,
543
+
$task_priority) {
540
544
541
545
$commit = new PhabricatorRepositoryCommit();
542
546
$conn_w = $repository->establishConnection('w');
···
559
563
$commit_identifier);
560
564
561
565
// After reviving a commit, schedule new daemons for it.
562
-
$this->didDiscoverCommit($repository, $commit, $epoch);
566
+
$this->didDiscoverCommit($repository, $commit, $epoch, $task_priority);
563
567
return;
564
568
}
565
569
···
620
624
}
621
625
$commit->saveTransaction();
622
626
623
-
$this->didDiscoverCommit($repository, $commit, $epoch);
627
+
$this->didDiscoverCommit($repository, $commit, $epoch, $task_priority);
624
628
625
629
if ($this->repairMode) {
626
630
// Normally, the query should throw a duplicate key exception. If we
···
648
652
private function didDiscoverCommit(
649
653
PhabricatorRepository $repository,
650
654
PhabricatorRepositoryCommit $commit,
651
-
$epoch) {
655
+
$epoch,
656
+
$task_priority) {
652
657
653
-
$this->insertTask($repository, $commit);
658
+
$this->insertTask($repository, $commit, $task_priority);
654
659
655
660
// Update the repository summary table.
656
661
queryfx(
···
677
682
private function insertTask(
678
683
PhabricatorRepository $repository,
679
684
PhabricatorRepositoryCommit $commit,
685
+
$task_priority,
680
686
$data = array()) {
681
687
682
688
$vcs = $repository->getVersionControlSystem();
···
695
701
}
696
702
697
703
$data['commitID'] = $commit->getID();
698
-
699
-
// If the repository is importing for the first time, we schedule tasks
700
-
// at IMPORT priority, which is very low. Making progress on importing a
701
-
// new repository for the first time is less important than any other
702
-
// daemon task.
703
-
704
-
// If the repository has finished importing and we're just catching up
705
-
// on recent commits, we schedule discovery at COMMIT priority, which is
706
-
// slightly below the default priority.
707
-
708
-
// Note that followup tasks and triggered tasks (like those generated by
709
-
// Herald or Harbormaster) will queue at DEFAULT priority, so that each
710
-
// commit tends to fully import before we start the next one. This tends
711
-
// to give imports fairly predictable progress. See T11677 for some
712
-
// discussion.
713
-
714
-
if ($repository->isImporting()) {
715
-
$task_priority = PhabricatorWorker::PRIORITY_IMPORT;
716
-
} else {
717
-
$task_priority = PhabricatorWorker::PRIORITY_COMMIT;
718
-
}
719
704
720
705
$options = array(
721
706
'priority' => $task_priority,
···
932
917
$data['N'],
933
918
$data['id'],
934
919
$data['epoch']);
920
+
}
921
+
922
+
private function getImportTaskPriority(
923
+
PhabricatorRepository $repository,
924
+
array $refs) {
925
+
926
+
// If the repository is importing for the first time, we schedule tasks
927
+
// at IMPORT priority, which is very low. Making progress on importing a
928
+
// new repository for the first time is less important than any other
929
+
// daemon task.
930
+
931
+
// If the repository has finished importing and we're just catching up
932
+
// on recent commits, we usually schedule discovery at COMMIT priority,
933
+
// which is slightly below the default priority.
934
+
935
+
// Note that followup tasks and triggered tasks (like those generated by
936
+
// Herald or Harbormaster) will queue at DEFAULT priority, so that each
937
+
// commit tends to fully import before we start the next one. This tends
938
+
// to give imports fairly predictable progress. See T11677 for some
939
+
// discussion.
940
+
941
+
if ($repository->isImporting()) {
942
+
$this->log(
943
+
pht(
944
+
'Importing %s commit(s) at low priority ("PRIORITY_IMPORT") '.
945
+
'because this repository is still importing.',
946
+
phutil_count($refs)));
947
+
948
+
return PhabricatorWorker::PRIORITY_IMPORT;
949
+
}
950
+
951
+
// See T13369. If we've discovered a lot of commits at once, import them
952
+
// at lower priority.
953
+
954
+
// This is mostly aimed at reducing the impact that synchronizing thousands
955
+
// of commits from a remote upstream has on other repositories. The queue
956
+
// is "mostly FIFO", so queueing a thousand commit imports can stall other
957
+
// repositories.
958
+
959
+
// In a perfect world we'd probably give repositories round-robin queue
960
+
// priority, but we don't currently have the primitives for this and there
961
+
// isn't a strong case for building them.
962
+
963
+
// Use "a whole lot of commits showed up at once" as a heuristic for
964
+
// detecting "someone synchronized an upstream", and import them at a lower
965
+
// priority to more closely approximate fair scheduling.
966
+
967
+
if (count($refs) >= PhabricatorRepository::LOWPRI_THRESHOLD) {
968
+
$this->log(
969
+
pht(
970
+
'Importing %s commit(s) at low priority ("PRIORITY_IMPORT") '.
971
+
'because many commits were discovered at once.',
972
+
phutil_count($refs)));
973
+
974
+
return PhabricatorWorker::PRIORITY_IMPORT;
975
+
}
976
+
977
+
// Otherwise, import at normal priority.
978
+
979
+
if ($refs) {
980
+
$this->log(
981
+
pht(
982
+
'Importing %s commit(s) at normal priority ("PRIORITY_COMMIT").',
983
+
phutil_count($refs)));
984
+
}
985
+
986
+
return PhabricatorWorker::PRIORITY_COMMIT;
935
987
}
936
988
937
989
}
+2
src/applications/repository/storage/PhabricatorRepository.php
+2
src/applications/repository/storage/PhabricatorRepository.php