@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Record parent relationships when discovering commits
Summary:
Ref T4455. This adds a `repository_parents` table which stores `<childCommitID, parentCommitID>` relationships.
For new commits, it is populated when commits are discovered.
For older commits, there's a `bin/repository parents` script to rebuild the data.
Right now, there's no UI suggestion that you should run the script. I haven't come up with a super clean way to do this, and this table will only improve performance for now, so it's not important that we get everyone to run the script right away. I'm just leaving it for the moment, and we can figure out how to tell admins to run it later.
The ultimate goal is to solve T2683, but solving T4455 gets us some stuff anyway (for example, we can serve `diffusion.commitparentsquery` faster out of this cache).
Test Plan:
- Used `bin/repository discover` to discover new commits in Git, SVN and Mercurial repositories.
- Used `bin/repository parents` to rebuild Git and Mercurial repositories (SVN repos just exit with a message).
- Verified that the table appears to be sensible.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: jhurwitz, epriestley
Maniphest Tasks: T4455
Differential Revision: https://secure.phabricator.com/D9044
+216
-4
+7
resources/sql/autopatches/20140512.dparents.1.sql
+7
resources/sql/autopatches/20140512.dparents.1.sql
···
1
+
CREATE TABLE {$NAMESPACE}_repository.repository_parents (
2
+
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
childCommitID INT UNSIGNED NOT NULL,
4
+
parentCommitID INT UNSIGNED NOT NULL,
5
+
UNIQUE `key_child` (childCommitID, parentCommitID),
6
+
KEY `key_parent` (parentCommitID)
7
+
) ENGINE=InnoDB, COLLATE utf8_general_ci;
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
1993
1993
'PhabricatorRepositoryManagementLookupUsersWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementLookupUsersWorkflow.php',
1994
1994
'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMarkImportedWorkflow.php',
1995
1995
'PhabricatorRepositoryManagementMirrorWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementMirrorWorkflow.php',
1996
+
'PhabricatorRepositoryManagementParentsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementParentsWorkflow.php',
1996
1997
'PhabricatorRepositoryManagementPullWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementPullWorkflow.php',
1997
1998
'PhabricatorRepositoryManagementRefsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementRefsWorkflow.php',
1998
1999
'PhabricatorRepositoryManagementUpdateWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php',
···
4835
4836
'PhabricatorRepositoryManagementLookupUsersWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4836
4837
'PhabricatorRepositoryManagementMarkImportedWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4837
4838
'PhabricatorRepositoryManagementMirrorWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4839
+
'PhabricatorRepositoryManagementParentsWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4838
4840
'PhabricatorRepositoryManagementPullWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4839
4841
'PhabricatorRepositoryManagementRefsWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
4840
4842
'PhabricatorRepositoryManagementUpdateWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
+10
src/applications/repository/engine/PhabricatorRepositoryCommitRef.php
+10
src/applications/repository/engine/PhabricatorRepositoryCommitRef.php
···
6
6
private $epoch;
7
7
private $branch;
8
8
private $canCloseImmediately;
9
+
private $parents = array();
9
10
10
11
public function setIdentifier($identifier) {
11
12
$this->identifier = $identifier;
···
41
42
42
43
public function getCanCloseImmediately() {
43
44
return $this->canCloseImmediately;
45
+
}
46
+
47
+
public function setParents(array $parents) {
48
+
$this->parents = $parents;
49
+
return $this;
50
+
}
51
+
52
+
public function getParents() {
53
+
return $this->parents;
44
54
}
45
55
46
56
}
+47
-4
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
+47
-4
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
···
61
61
$repository,
62
62
$ref->getIdentifier(),
63
63
$ref->getEpoch(),
64
-
$ref->getCanCloseImmediately());
64
+
$ref->getCanCloseImmediately(),
65
+
$ref->getParents());
65
66
66
67
$this->commitCache[$ref->getIdentifier()] = true;
67
68
}
···
436
437
$refs[] = id(new PhabricatorRepositoryCommitRef())
437
438
->setIdentifier($commit)
438
439
->setEpoch($stream->getCommitDate($commit))
439
-
->setCanCloseImmediately($close_immediately);
440
+
->setCanCloseImmediately($close_immediately)
441
+
->setParents($stream->getParents($commit));
440
442
}
441
443
442
444
return $refs;
···
534
536
PhabricatorRepository $repository,
535
537
$commit_identifier,
536
538
$epoch,
537
-
$close_immediately) {
539
+
$close_immediately,
540
+
array $parents) {
538
541
539
542
$commit = new PhabricatorRepositoryCommit();
540
543
$commit->setRepositoryID($repository->getID());
···
546
549
547
550
$data = new PhabricatorRepositoryCommitData();
548
551
552
+
$conn_w = $repository->establishConnection('w');
553
+
549
554
try {
555
+
556
+
// If this commit has parents, look up their IDs. The parent commits
557
+
// should always exist already.
558
+
559
+
$parent_ids = array();
560
+
if ($parents) {
561
+
$parent_rows = queryfx_all(
562
+
$conn_w,
563
+
'SELECT id, commitIdentifier FROM %T
564
+
WHERE commitIdentifier IN (%Ls) AND repositoryID = %d',
565
+
$commit->getTableName(),
566
+
$parents,
567
+
$repository->getID());
568
+
569
+
$parent_map = ipull($parent_rows, 'id', 'commitIdentifier');
570
+
571
+
foreach ($parents as $parent) {
572
+
if (empty($parent_map[$parent])) {
573
+
throw new Exception(
574
+
pht('Unable to identify parent "%s"!', $parent));
575
+
}
576
+
$parent_ids[] = $parent_map[$parent];
577
+
}
578
+
}
579
+
550
580
$commit->openTransaction();
551
581
$commit->save();
582
+
552
583
$data->setCommitID($commit->getID());
553
584
$data->save();
585
+
586
+
foreach ($parent_ids as $parent_id) {
587
+
queryfx(
588
+
$conn_w,
589
+
'INSERT IGNORE INTO %T (childCommitID, parentCommitID)
590
+
VALUES (%d, %d)',
591
+
PhabricatorRepository::TABLE_PARENTS,
592
+
$commit->getID(),
593
+
$parent_id);
594
+
}
554
595
$commit->saveTransaction();
555
596
556
597
$this->insertTask($repository, $commit);
557
598
558
599
queryfx(
559
-
$repository->establishConnection('w'),
600
+
$conn_w,
560
601
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
561
602
VALUES (%d, 1, %d, %d)
562
603
ON DUPLICATE KEY UPDATE
···
582
623
'repository' => $repository,
583
624
'commit' => $commit,
584
625
)));
626
+
627
+
585
628
586
629
} catch (AphrontQueryDuplicateKeyException $ex) {
587
630
$commit->killTransaction();
+149
src/applications/repository/management/PhabricatorRepositoryManagementParentsWorkflow.php
+149
src/applications/repository/management/PhabricatorRepositoryManagementParentsWorkflow.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRepositoryManagementParentsWorkflow
4
+
extends PhabricatorRepositoryManagementWorkflow {
5
+
6
+
public function didConstruct() {
7
+
$this
8
+
->setName('parents')
9
+
->setExamples('**parents** [options] [__repository__] ...')
10
+
->setSynopsis(
11
+
pht(
12
+
'Build parent caches in repositories that are missing the data, '.
13
+
'or rebuild them in a specific __repository__.'))
14
+
->setArguments(
15
+
array(
16
+
array(
17
+
'name' => 'repos',
18
+
'wildcard' => true,
19
+
),
20
+
));
21
+
}
22
+
23
+
public function execute(PhutilArgumentParser $args) {
24
+
$repos = $this->loadRepositories($args, 'repos');
25
+
if (!$repos) {
26
+
$repos = id(new PhabricatorRepositoryQuery())
27
+
->setViewer($this->getViewer())
28
+
->execute();
29
+
}
30
+
31
+
$console = PhutilConsole::getConsole();
32
+
foreach ($repos as $repo) {
33
+
$monogram = $repo->getMonogram();
34
+
if ($repo->isSVN()) {
35
+
$console->writeOut(
36
+
"%s\n",
37
+
pht(
38
+
'Skipping "%s": Subversion repositories do not require this '.
39
+
'cache to be built.',
40
+
$monogram));
41
+
continue;
42
+
}
43
+
$this->rebuildRepository($repo);
44
+
}
45
+
46
+
return 0;
47
+
}
48
+
49
+
private function rebuildRepository(PhabricatorRepository $repo) {
50
+
$console = PhutilConsole::getConsole();
51
+
$console->writeOut("%s\n", pht('Rebuilding "%s"...', $repo->getMonogram()));
52
+
53
+
$refs = id(new PhabricatorRepositoryRefCursorQuery())
54
+
->setViewer($this->getViewer())
55
+
->withRefTypes(array(PhabricatorRepositoryRefCursor::TYPE_BRANCH))
56
+
->withRepositoryPHIDs(array($repo->getPHID()))
57
+
->execute();
58
+
59
+
$graph = array();
60
+
foreach ($refs as $ref) {
61
+
$console->writeOut(
62
+
"%s\n",
63
+
pht('Rebuilding branch "%s"...', $ref->getRefName()));
64
+
65
+
$commit = $ref->getCommitIdentifier();
66
+
67
+
if ($repo->isGit()) {
68
+
$stream = new PhabricatorGitGraphStream($repo, $commit);
69
+
} else {
70
+
$stream = new PhabricatorMercurialGraphStream($repo, $commit);
71
+
}
72
+
73
+
$discover = array($commit);
74
+
while ($discover) {
75
+
$target = array_pop($discover);
76
+
if (isset($graph[$target])) {
77
+
continue;
78
+
}
79
+
$graph[$target] = $stream->getParents($target);
80
+
foreach ($graph[$target] as $parent) {
81
+
$discover[] = $parent;
82
+
}
83
+
}
84
+
}
85
+
86
+
$console->writeOut(
87
+
"%s\n",
88
+
pht(
89
+
'Found %s total commit(s); updating...',
90
+
new PhutilNumber(count($graph))));
91
+
92
+
$commit_table = id(new PhabricatorRepositoryCommit());
93
+
$commit_table_name = $commit_table->getTableName();
94
+
$conn_w = $commit_table->establishConnection('w');
95
+
96
+
$bar = id(new PhutilConsoleProgressBar())
97
+
->setTotal(count($graph));
98
+
99
+
foreach ($graph as $child => $parents) {
100
+
$names = $parents;
101
+
$names[] = $child;
102
+
103
+
$rows = queryfx_all(
104
+
$conn_w,
105
+
'SELECT id, commitIdentifier FROM %T
106
+
WHERE commitIdentifier IN (%Ls) AND repositoryID = %d',
107
+
$commit_table_name,
108
+
$names,
109
+
$repo->getID());
110
+
111
+
$map = ipull($rows, 'id', 'commitIdentifier');
112
+
foreach ($names as $name) {
113
+
if (empty($map[$name])) {
114
+
throw new Exception(pht('Unknown commit "%s"!', $name));
115
+
}
116
+
}
117
+
118
+
$sql = array();
119
+
foreach ($parents as $parent) {
120
+
$sql[] = qsprintf(
121
+
$conn_w,
122
+
'(%d, %d)',
123
+
$map[$child],
124
+
$map[$parent]);
125
+
}
126
+
127
+
$commit_table->openTransaction();
128
+
queryfx(
129
+
$conn_w,
130
+
'DELETE FROM %T WHERE childCommitID = %d',
131
+
PhabricatorRepository::TABLE_PARENTS,
132
+
$map[$child]);
133
+
134
+
if ($sql) {
135
+
queryfx(
136
+
$conn_w,
137
+
'INSERT INTO %T (childCommitID, parentCommitID) VALUES %Q',
138
+
PhabricatorRepository::TABLE_PARENTS,
139
+
implode(', ', $sql));
140
+
}
141
+
$commit_table->saveTransaction();
142
+
143
+
$bar->update(1);
144
+
}
145
+
146
+
$bar->done();
147
+
}
148
+
149
+
}
+1
src/applications/repository/storage/PhabricatorRepository.php
+1
src/applications/repository/storage/PhabricatorRepository.php