@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Fetch and discover all Git ref types, not just branches
Summary:
Ref T9028. Fixes T6878. Currently, we only fetch and discover branches. This is fine 99% of the time but sometimes commits are pushed to just a tag, e.g.:
```
git checkout <some hash>
nano file.c
git commit -am '...'
git tag wild-wild-west
git push origin wild-wild-west
```
Through a similar process, commits can also be pushed to some arbitrary named ref (we do this for staging areas).
With the current rules, we don't fetch tag refs and won't discover these commits.
Change the rules so:
- we fetch all refs; and
- we discover ancestors of all refs.
Autoclose rules for tags and arbitrary refs are just hard-coded for now. We might make these more flexible in the future, or we might do forks instead, or maybe we'll have to do both.
Test Plan:
Pushed a commit to a tag ONLY (`vegetable1`).
<https://github.com/epriestley/poems/commit/cf508b8de62936ad985bdd0df86aeb206df7354a>
On `master`, prior to the change:
- Used `update` + `refs` + `discover`.
- Verified tag was not fetched with `git for-each-ref` in local working copy and the web UI.
- Verified commit was not discovered using the web UI.
With this patch applied:
- Used `update`, saw a `refs/*` fetch instead of a `refs/heads/*` fetch.
- Used `git for-each-ref` to verify that tag fetched.
- Used `repository refs`.
- Saw new tag appear in the tags list in the web UI.
- Saw new refcursor appear in refcursor table.
- Used `repository discover --verbose` and examine refs for sanity.
- Saw commit row appear in database.
- Saw commit skeleton appear in web UI.
- Ran `bin/phd debug task`.
- Saw commit fully parse.
{F1689319}
Reviewers: chad
Reviewed By: chad
Subscribers: avivey
Maniphest Tasks: T6878, T9028
Differential Revision: https://secure.phabricator.com/D16129
+160
-129
+17
src/applications/diffusion/data/DiffusionRepositoryRef.php
+17
src/applications/diffusion/data/DiffusionRepositoryRef.php
···
7
7
8
8
private $shortName;
9
9
private $commitIdentifier;
10
+
private $refType;
10
11
private $rawFields = array();
11
12
12
13
public function setRawFields(array $raw_fields) {
···
36
37
return $this->shortName;
37
38
}
38
39
40
+
public function setRefType($ref_type) {
41
+
$this->refType = $ref_type;
42
+
return $this;
43
+
}
44
+
45
+
public function getRefType() {
46
+
return $this->refType;
47
+
}
48
+
49
+
public function isBranch() {
50
+
$type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH;
51
+
return ($this->getRefType() === $type_branch);
52
+
}
53
+
39
54
40
55
/* -( Serialization )------------------------------------------------------ */
41
56
···
44
59
return array(
45
60
'shortName' => $this->shortName,
46
61
'commitIdentifier' => $this->commitIdentifier,
62
+
'refType' => $this->refType,
47
63
'rawFields' => $this->rawFields,
48
64
);
49
65
}
···
52
68
return id(new DiffusionRepositoryRef())
53
69
->setShortName($dict['shortName'])
54
70
->setCommitIdentifier($dict['commitIdentifier'])
71
+
->setRefType($dict['refType'])
55
72
->setRawFields($dict['rawFields']);
56
73
}
57
74
+81
-65
src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php
+81
-65
src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php
···
14
14
}
15
15
16
16
protected function executeQuery() {
17
+
$type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH;
18
+
$type_tag = PhabricatorRepositoryRefCursor::TYPE_TAG;
19
+
$type_ref = PhabricatorRepositoryRefCursor::TYPE_REF;
20
+
17
21
$ref_types = $this->refTypes;
18
-
if ($ref_types) {
19
-
$type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH;
20
-
$type_tag = PhabricatorRepositoryRefCursor::TYPE_TAG;
22
+
if (!$ref_types) {
23
+
$ref_types = array($type_branch, $type_tag, $type_ref);
24
+
}
21
25
22
-
$ref_types = array_fuse($ref_types);
26
+
$ref_types = array_fuse($ref_types);
23
27
24
-
$with_branches = isset($ref_types[$type_branch]);
25
-
$with_tags = isset($ref_types[$type_tag]);
26
-
} else {
27
-
$with_branches = true;
28
-
$with_tags = true;
29
-
}
28
+
$with_branches = isset($ref_types[$type_branch]);
29
+
$with_tags = isset($ref_types[$type_tag]);
30
+
$with_refs = isset($refs_types[$type_ref]);
30
31
31
32
$repository = $this->getRepository();
32
33
33
34
$prefixes = array();
34
35
35
-
if ($with_branches) {
36
-
if ($repository->isWorkingCopyBare()) {
37
-
$prefix = 'refs/heads/';
38
-
} else {
39
-
$remote = DiffusionGitBranch::DEFAULT_GIT_REMOTE;
40
-
$prefix = 'refs/remotes/'.$remote.'/';
41
-
}
42
-
$prefixes[] = $prefix;
36
+
if ($repository->isWorkingCopyBare()) {
37
+
$branch_prefix = 'refs/heads/';
38
+
} else {
39
+
$remote = DiffusionGitBranch::DEFAULT_GIT_REMOTE;
40
+
$branch_prefix = 'refs/remotes/'.$remote.'/';
43
41
}
44
42
45
-
if ($with_tags) {
46
-
$prefixes[] = 'refs/tags/';
43
+
$tag_prefix = 'refs/tags/';
44
+
45
+
46
+
if ($with_refs || count($ref_types) > 1) {
47
+
// If we're loading refs or more than one type of ref, just query
48
+
// everything.
49
+
$prefix = 'refs/';
50
+
} else {
51
+
if ($with_branches) {
52
+
$prefix = $branch_prefix;
53
+
}
54
+
if ($with_tags) {
55
+
$prefix = $tag_prefix;
56
+
}
47
57
}
48
58
49
-
$order = '-creatordate';
59
+
$branch_len = strlen($branch_prefix);
60
+
$tag_len = strlen($tag_prefix);
50
61
51
-
$futures = array();
52
-
foreach ($prefixes as $prefix) {
53
-
$futures[$prefix] = $repository->getLocalCommandFuture(
54
-
'for-each-ref --sort=%s --format=%s %s',
55
-
$order,
56
-
$this->getFormatString(),
57
-
$prefix);
58
-
}
62
+
list($stdout) = $repository->execxLocalCommand(
63
+
'for-each-ref --sort=%s --format=%s -- %s',
64
+
'-creatordate',
65
+
$this->getFormatString(),
66
+
$prefix);
59
67
60
-
// Resolve all the futures first. We want to iterate over them in prefix
61
-
// order, not resolution order.
62
-
foreach (new FutureIterator($futures) as $prefix => $future) {
63
-
$future->resolvex();
68
+
$stdout = rtrim($stdout);
69
+
if (!strlen($stdout)) {
70
+
return array();
64
71
}
65
72
73
+
// NOTE: Although git supports --count, we can't apply any offset or
74
+
// limit logic until the very end because we may encounter a HEAD which
75
+
// we want to discard.
76
+
77
+
$lines = explode("\n", $stdout);
66
78
$results = array();
67
-
foreach ($futures as $prefix => $future) {
68
-
list($stdout) = $future->resolvex();
79
+
foreach ($lines as $line) {
80
+
$fields = $this->extractFields($line);
81
+
82
+
$refname = $fields['refname'];
83
+
if (!strncmp($refname, $branch_prefix, $branch_len)) {
84
+
$short = substr($refname, $branch_len);
85
+
$type = $type_branch;
86
+
} else if (!strncmp($refname, $tag_prefix, $tag_len)) {
87
+
$short = substr($refname, $tag_len);
88
+
$type = $type_tag;
89
+
} else {
90
+
$short = $refname;
91
+
$type = $type_ref;
92
+
}
69
93
70
-
$stdout = rtrim($stdout);
71
-
if (!strlen($stdout)) {
94
+
// If this isn't a type of ref we care about, skip it.
95
+
if (empty($ref_types[$type])) {
72
96
continue;
73
97
}
74
98
75
-
// NOTE: Although git supports --count, we can't apply any offset or
76
-
// limit logic until the very end because we may encounter a HEAD which
77
-
// we want to discard.
99
+
// If this is the local HEAD, skip it.
100
+
if ($short == 'HEAD') {
101
+
continue;
102
+
}
78
103
79
-
$lines = explode("\n", $stdout);
80
-
foreach ($lines as $line) {
81
-
$fields = $this->extractFields($line);
82
-
83
-
$creator = $fields['creator'];
84
-
$matches = null;
85
-
if (preg_match('/^(.*) ([0-9]+) ([0-9+-]+)$/', $creator, $matches)) {
86
-
$fields['author'] = $matches[1];
87
-
$fields['epoch'] = (int)$matches[2];
88
-
} else {
89
-
$fields['author'] = null;
90
-
$fields['epoch'] = null;
91
-
}
92
-
93
-
$commit = nonempty($fields['*objectname'], $fields['objectname']);
104
+
$creator = $fields['creator'];
105
+
$matches = null;
106
+
if (preg_match('/^(.*) ([0-9]+) ([0-9+-]+)$/', $creator, $matches)) {
107
+
$fields['author'] = $matches[1];
108
+
$fields['epoch'] = (int)$matches[2];
109
+
} else {
110
+
$fields['author'] = null;
111
+
$fields['epoch'] = null;
112
+
}
94
113
95
-
$short = substr($fields['refname'], strlen($prefix));
96
-
if ($short == 'HEAD') {
97
-
continue;
98
-
}
114
+
$commit = nonempty($fields['*objectname'], $fields['objectname']);
99
115
100
-
$ref = id(new DiffusionRepositoryRef())
101
-
->setShortName($short)
102
-
->setCommitIdentifier($commit)
103
-
->setRawFields($fields);
116
+
$ref = id(new DiffusionRepositoryRef())
117
+
->setRefType($type)
118
+
->setShortName($short)
119
+
->setCommitIdentifier($commit)
120
+
->setRawFields($fields);
104
121
105
-
$results[] = $ref;
106
-
}
122
+
$results[] = $ref;
107
123
}
108
124
109
125
return $results;
+27
-33
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
+27
-33
src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
···
130
130
$this->verifyGitOrigin($repository);
131
131
}
132
132
133
-
// TODO: This should also import tags, but some of the logic is still
134
-
// branch-specific today.
135
-
136
-
$branches = id(new DiffusionLowLevelGitRefQuery())
133
+
$heads = id(new DiffusionLowLevelGitRefQuery())
137
134
->setRepository($repository)
138
-
->withRefTypes(
139
-
array(
140
-
PhabricatorRepositoryRefCursor::TYPE_BRANCH,
141
-
))
142
135
->execute();
143
136
144
-
if (!$branches) {
145
-
// This repository has no branches at all, so we don't need to do
137
+
if (!$heads) {
138
+
// This repository has no heads at all, so we don't need to do
146
139
// anything. Generally, this means the repository is empty.
147
140
return array();
148
141
}
149
142
150
-
$branches = $this->sortBranches($branches);
151
-
$branches = mpull($branches, 'getCommitIdentifier', 'getShortName');
143
+
$heads = $this->sortRefs($heads);
144
+
$head_commits = mpull($heads, 'getCommitIdentifier');
152
145
153
146
$this->log(
154
147
pht(
155
148
'Discovering commits in repository "%s".',
156
149
$repository->getDisplayName()));
157
150
158
-
$this->fillCommitCache(array_values($branches));
151
+
$this->fillCommitCache($head_commits);
159
152
160
153
$refs = array();
161
-
foreach ($branches as $name => $commit) {
162
-
$this->log(pht('Examining branch "%s", at "%s".', $name, $commit));
154
+
foreach ($heads as $ref) {
155
+
$name = $ref->getShortName();
156
+
$commit = $ref->getCommitIdentifier();
163
157
164
-
if (!$repository->shouldTrackBranch($name)) {
165
-
$this->log(pht('Skipping, branch is untracked.'));
158
+
$this->log(pht('Examining ref "%s", at "%s".', $name, $commit));
159
+
160
+
if (!$repository->shouldTrackRef($ref)) {
161
+
$this->log(pht('Skipping, ref is untracked.'));
166
162
continue;
167
163
}
168
164
···
173
169
174
170
$this->log(pht('Looking for new commits.'));
175
171
176
-
$branch_refs = $this->discoverStreamAncestry(
172
+
$head_refs = $this->discoverStreamAncestry(
177
173
new PhabricatorGitGraphStream($repository, $commit),
178
174
$commit,
179
-
$repository->shouldAutocloseBranch($name));
175
+
$repository->shouldAutocloseRef($ref));
180
176
181
-
$this->didDiscoverRefs($branch_refs);
177
+
$this->didDiscoverRefs($head_refs);
182
178
183
-
$refs[] = $branch_refs;
179
+
$refs[] = $head_refs;
184
180
}
185
181
186
182
return array_mergev($refs);
···
469
465
*
470
466
* @task internal
471
467
*
472
-
* @param list<DiffusionRepositoryRef> List of branch heads.
473
-
* @return list<DiffusionRepositoryRef> Sorted list of branch heads.
468
+
* @param list<DiffusionRepositoryRef> List of refs.
469
+
* @return list<DiffusionRepositoryRef> Sorted list of refs.
474
470
*/
475
-
private function sortBranches(array $branches) {
471
+
private function sortRefs(array $refs) {
476
472
$repository = $this->getRepository();
477
473
478
-
$head_branches = array();
479
-
$tail_branches = array();
480
-
foreach ($branches as $branch) {
481
-
$name = $branch->getShortName();
482
-
483
-
if ($repository->shouldAutocloseBranch($name)) {
484
-
$head_branches[] = $branch;
474
+
$head_refs = array();
475
+
$tail_refs = array();
476
+
foreach ($refs as $ref) {
477
+
if ($repository->shouldAutocloseRef($ref)) {
478
+
$head_refs[] = $ref;
485
479
} else {
486
-
$tail_branches[] = $branch;
480
+
$tail_refs[] = $ref;
487
481
}
488
482
}
489
483
490
-
return array_merge($head_branches, $tail_branches);
484
+
return array_merge($head_refs, $tail_refs);
491
485
}
492
486
493
487
+1
-1
src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
+1
-1
src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
···
347
347
// For bare working copies, we need this magic incantation.
348
348
$future = $repository->getRemoteCommandFuture(
349
349
'fetch origin %s --prune',
350
-
'+refs/heads/*:refs/heads/*');
350
+
'+refs/*:refs/*');
351
351
} else {
352
352
$future = $repository->getRemoteCommandFuture(
353
353
'fetch --all --prune');
+17
-30
src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
+17
-30
src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
···
25
25
switch ($vcs) {
26
26
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
27
27
// No meaningful refs of any type in Subversion.
28
-
$branches = array();
29
-
$bookmarks = array();
30
-
$tags = array();
28
+
$maps = array();
31
29
break;
32
30
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
33
31
$branches = $this->loadMercurialBranchPositions($repository);
34
32
$bookmarks = $this->loadMercurialBookmarkPositions($repository);
35
-
$tags = array();
33
+
$maps = array(
34
+
PhabricatorRepositoryRefCursor::TYPE_BRANCH => $branches,
35
+
PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => $bookmarks,
36
+
);
37
+
36
38
$branches_may_close = true;
37
39
break;
38
40
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
39
-
$branches = $this->loadGitBranchPositions($repository);
40
-
$bookmarks = array();
41
-
$tags = $this->loadGitTagPositions($repository);
41
+
$maps = $this->loadGitRefPositions($repository);
42
42
break;
43
43
default:
44
44
throw new Exception(pht('Unknown VCS "%s"!', $vcs));
45
45
}
46
46
47
-
$maps = array(
48
-
PhabricatorRepositoryRefCursor::TYPE_BRANCH => $branches,
49
-
PhabricatorRepositoryRefCursor::TYPE_TAG => $tags,
50
-
PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => $bookmarks,
47
+
// Fill in any missing types with empty lists.
48
+
$maps = $maps + array(
49
+
PhabricatorRepositoryRefCursor::TYPE_BRANCH => array(),
50
+
PhabricatorRepositoryRefCursor::TYPE_TAG => array(),
51
+
PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => array(),
52
+
PhabricatorRepositoryRefCursor::TYPE_REF => array(),
51
53
);
52
54
53
55
$all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
···
91
93
$this->deadRefs = array();
92
94
}
93
95
96
+
$branches = $maps[PhabricatorRepositoryRefCursor::TYPE_BRANCH];
94
97
if ($branches && $branches_may_close) {
95
98
$this->updateBranchStates($repository, $branches);
96
99
}
···
449
452
/**
450
453
* @task git
451
454
*/
452
-
private function loadGitBranchPositions(PhabricatorRepository $repository) {
453
-
return id(new DiffusionLowLevelGitRefQuery())
455
+
private function loadGitRefPositions(PhabricatorRepository $repository) {
456
+
$refs = id(new DiffusionLowLevelGitRefQuery())
454
457
->setRepository($repository)
455
-
->withRefTypes(
456
-
array(
457
-
PhabricatorRepositoryRefCursor::TYPE_BRANCH,
458
-
))
459
458
->execute();
460
-
}
461
459
462
-
463
-
/**
464
-
* @task git
465
-
*/
466
-
private function loadGitTagPositions(PhabricatorRepository $repository) {
467
-
return id(new DiffusionLowLevelGitRefQuery())
468
-
->setRepository($repository)
469
-
->withRefTypes(
470
-
array(
471
-
PhabricatorRepositoryRefCursor::TYPE_TAG,
472
-
))
473
-
->execute();
460
+
return mgroup($refs, 'getRefType');
474
461
}
475
462
476
463
+16
src/applications/repository/storage/PhabricatorRepository.php
+16
src/applications/repository/storage/PhabricatorRepository.php
···
910
910
return null;
911
911
}
912
912
913
+
public function shouldTrackRef(DiffusionRepositoryRef $ref) {
914
+
if (!$ref->isBranch()) {
915
+
return true;
916
+
}
917
+
918
+
return $this->shouldTrackBranch($ref->getShortName());
919
+
}
920
+
913
921
public function shouldTrackBranch($branch) {
914
922
return $this->isBranchInFilter($branch, 'branch-filter');
915
923
}
···
1019
1027
1020
1028
/* -( Autoclose )---------------------------------------------------------- */
1021
1029
1030
+
1031
+
public function shouldAutocloseRef(DiffusionRepositoryRef $ref) {
1032
+
if (!$ref->isBranch()) {
1033
+
return false;
1034
+
}
1035
+
1036
+
return $this->shouldAutocloseBranch($ref->getShortName());
1037
+
}
1022
1038
1023
1039
/**
1024
1040
* Determine if autoclose is active for a branch.