@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Batch execution of LastModified query
Summary:
Ref T2683. Further reduces query count of last modified loads; we're now at 11 instead of 200+.
(This works in SVN but could be further optimized.)
Test Plan:
Loaded SVN, Mercurial, Git:
{F34864}
{F34865}
{F34866}
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley, vrana, aran
Maniphest Tasks: T2683
Differential Revision: https://secure.phabricator.com/D5256
+88
-119
+10
-16
src/applications/diffusion/conduit/ConduitAPI_diffusion_diffquery_Method.php
+10
-16
src/applications/diffusion/conduit/ConduitAPI_diffusion_diffquery_Method.php
···
30
30
31
31
return array(
32
32
'changes' => mpull($result, 'toDictionary'),
33
-
'effectiveCommit' => $this->getEffectiveCommit($request));
33
+
'effectiveCommit' => $this->getEffectiveCommit($request),
34
+
);
34
35
}
35
36
36
37
protected function getGitResult(ConduitAPIRequest $request) {
···
160
161
private function getEffectiveCommit(ConduitAPIRequest $request) {
161
162
if ($this->effectiveCommit === null) {
162
163
$drequest = $this->getDiffusionRequest();
163
-
$user = $request->getUser();
164
-
$commit = null;
165
164
166
-
$conduit_result = DiffusionQuery::callConduitWithDiffusionRequest(
167
-
$user,
165
+
$path = $drequest->getPath();
166
+
$result = DiffusionQuery::callConduitWithDiffusionRequest(
167
+
$request->getUser(),
168
168
$drequest,
169
169
'diffusion.lastmodifiedquery',
170
170
array(
171
-
'commit' => $drequest->getCommit(),
172
-
'path' => $drequest->getPath()));
173
-
$c_dict = $conduit_result['commit'];
174
-
if ($c_dict) {
175
-
$commit = PhabricatorRepositoryCommit::newFromDictionary($c_dict);
176
-
}
177
-
if (!$commit) {
178
-
// TODO: Improve error messages here.
179
-
return null;
180
-
}
181
-
$this->effectiveCommit = $commit->getCommitIdentifier();
171
+
'paths' => array($path => $drequest->getCommit()),
172
+
));
173
+
174
+
$this->effectiveCommit = idx($result, $path);
182
175
}
176
+
183
177
return $this->effectiveCommit;
184
178
}
185
179
+43
-74
src/applications/diffusion/conduit/ConduitAPI_diffusion_lastmodifiedquery_Method.php
+43
-74
src/applications/diffusion/conduit/ConduitAPI_diffusion_lastmodifiedquery_Method.php
···
1
1
<?php
2
2
3
-
/**
4
-
* @group conduit
5
-
*/
6
3
final class ConduitAPI_diffusion_lastmodifiedquery_Method
7
4
extends ConduitAPI_diffusion_abstractquery_Method {
8
5
9
6
public function getMethodDescription() {
10
-
return
11
-
'Get last modified information from a repository for a specific commit '.
12
-
'at a specific path.';
7
+
return pht('Get the commits at which paths were last modified.');
13
8
}
14
9
15
10
public function defineReturnType() {
16
-
return 'array';
11
+
return 'map<string, string>';
17
12
}
18
13
19
14
protected function defineCustomParamTypes() {
20
15
return array(
21
-
'commit' => 'required string',
22
-
'path' => 'required string',
16
+
'paths' => 'required map<string, string>',
23
17
);
24
18
}
25
19
26
-
protected function getResult(ConduitAPIRequest $request) {
27
-
list($commit, $commit_data) = parent::getResult($request);
28
-
if ($commit) {
29
-
$commit = $commit->toDictionary();
30
-
}
31
-
if ($commit_data) {
32
-
$commit_data = $commit_data->toDictionary();
33
-
}
34
-
return array(
35
-
'commit' => $commit,
36
-
'commitData' => $commit_data);
37
-
}
38
-
39
20
protected function getGitResult(ConduitAPIRequest $request) {
40
21
$drequest = $this->getDiffusionRequest();
41
22
$repository = $drequest->getRepository();
42
23
43
-
list($hash) = $repository->execxLocalCommand(
44
-
'log -n1 --format=%%H %s -- %s',
45
-
$drequest->getCommit(),
46
-
$drequest->getPath());
47
-
$hash = trim($hash);
24
+
$result = array();
25
+
foreach ($request->getValue('paths') as $path => $commit) {
26
+
list($hash) = $repository->execxLocalCommand(
27
+
'log -n1 --format=%%H %s -- %s',
28
+
$commit,
29
+
$path);
30
+
$result[$path] = trim($hash);
31
+
}
48
32
49
-
return $this->loadDataFromHash($hash);
33
+
return $result;
50
34
}
51
35
52
36
protected function getSVNResult(ConduitAPIRequest $request) {
53
37
$drequest = $this->getDiffusionRequest();
54
38
$repository = $drequest->getRepository();
55
39
56
-
$path = $drequest->getPath();
40
+
$result = array();
41
+
foreach ($request->getValue('paths') as $path => $commit) {
42
+
$history_result = DiffusionQuery::callConduitWithDiffusionRequest(
43
+
$request->getUser(),
44
+
$drequest,
45
+
'diffusion.historyquery',
46
+
array(
47
+
'commit' => $commit,
48
+
'path' => $path,
49
+
'limit' => 1,
50
+
'offset' => 0,
51
+
'needDirectChanges' => true,
52
+
'needChildChanges' => true,
53
+
));
57
54
58
-
$history_result = DiffusionQuery::callConduitWithDiffusionRequest(
59
-
$request->getUser(),
60
-
$drequest,
61
-
'diffusion.historyquery',
62
-
array(
63
-
'commit' => $drequest->getCommit(),
64
-
'path' => $path,
65
-
'limit' => 1,
66
-
'offset' => 0,
67
-
'needDirectChanges' => true,
68
-
'needChildChanges' => true));
69
-
$history_array = DiffusionPathChange::newFromConduit(
70
-
$history_result['pathChanges']);
71
-
72
-
if (!$history_array) {
73
-
return array(array(), array());
55
+
$history_array = DiffusionPathChange::newFromConduit(
56
+
$history_result['pathChanges']);
57
+
if ($history_array) {
58
+
$result[$path] = head($history_array)
59
+
->getCommit()
60
+
->getCommitIdentifier();
61
+
}
74
62
}
75
63
76
-
$history = reset($history_array);
77
-
78
-
return array($history->getCommit(), $history->getCommitData());
64
+
return $result;
79
65
}
80
66
81
67
protected function getMercurialResult(ConduitAPIRequest $request) {
82
68
$drequest = $this->getDiffusionRequest();
83
69
$repository = $drequest->getRepository();
84
70
85
-
$path = $drequest->getPath();
86
-
87
-
list($hash) = $repository->execxLocalCommand(
88
-
'log --template %s --limit 1 --removed --rev %s -- %s',
89
-
'{node}',
90
-
hgsprintf('reverse(ancestors(%s))', $drequest->getCommit()),
91
-
nonempty(ltrim($path, '/'), '.'));
92
-
93
-
return $this->loadDataFromHash($hash);
94
-
}
95
-
96
-
private function loadDataFromHash($hash) {
97
-
$drequest = $this->getDiffusionRequest();
98
-
$repository = $drequest->getRepository();
99
-
100
-
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
101
-
'repositoryID = %d AND commitIdentifier = %s',
102
-
$repository->getID(),
103
-
$hash);
104
-
105
-
if ($commit) {
106
-
$commit_data = $commit->loadCommitData();
107
-
} else {
108
-
$commit = array();
109
-
$commit_data = array();
71
+
$result = array();
72
+
foreach ($request->getValue('paths') as $path => $commit) {
73
+
list($hash) = $repository->execxLocalCommand(
74
+
'log --template %s --limit 1 --removed --rev %s -- %s',
75
+
'{node}',
76
+
hgsprintf('reverse(ancestors(%s))', $commit),
77
+
nonempty(ltrim($path, '/'), '.'));
78
+
$result[$path] = trim($hash);
110
79
}
111
80
112
-
return array($commit, $commit_data);
81
+
return $result;
113
82
}
114
83
115
84
}
+32
-29
src/applications/diffusion/controller/DiffusionLastModifiedController.php
+32
-29
src/applications/diffusion/controller/DiffusionLastModifiedController.php
···
9
9
public function processRequest() {
10
10
$drequest = $this->getDiffusionRequest();
11
11
$request = $this->getRequest();
12
+
$viewer = $request->getUser();
12
13
13
14
$paths = $request->getStr('paths');
14
15
$paths = json_decode($paths, true);
···
16
17
return new Aphront400Response();
17
18
}
18
19
20
+
$modified_map = $this->callConduitWithDiffusionRequest(
21
+
'diffusion.lastmodifiedquery',
22
+
array(
23
+
'paths' => array_fill_keys($paths, $drequest->getCommit()),
24
+
));
25
+
26
+
if ($modified_map) {
27
+
$commit_map = id(new DiffusionCommitQuery())
28
+
->setViewer($viewer)
29
+
->withRepository($drequest->getRepository())
30
+
->withIdentifiers(array_values($modified_map))
31
+
->needCommitData(true)
32
+
->execute();
33
+
$commit_map = mpull($commit_map, null, 'getCommitIdentifier');
34
+
} else {
35
+
$commit_map = array();
36
+
}
37
+
19
38
$commits = array();
20
39
foreach ($paths as $path) {
21
-
$prequest = clone $drequest;
22
-
$prequest->setPath($path);
23
-
24
-
$conduit_result = $this->callConduitWithDiffusionRequest(
25
-
'diffusion.lastmodifiedquery',
26
-
array(
27
-
'commit' => $prequest->getCommit(),
28
-
'path' => $prequest->getPath(),
29
-
));
30
-
31
-
$commit = PhabricatorRepositoryCommit::newFromDictionary(
32
-
$conduit_result['commit']);
33
-
34
-
$commit_data = PhabricatorRepositoryCommitData::newFromDictionary(
35
-
$conduit_result['commitData']);
36
-
37
-
$commit->attachCommitData($commit_data);
38
-
39
-
$phids = array();
40
-
if ($commit_data) {
41
-
if ($commit_data->getCommitDetail('authorPHID')) {
42
-
$phids[$commit_data->getCommitDetail('authorPHID')] = true;
43
-
}
44
-
if ($commit_data->getCommitDetail('committerPHID')) {
45
-
$phids[$commit_data->getCommitDetail('committerPHID')] = true;
40
+
$modified_at = idx($modified_map, $path);
41
+
if ($modified_at) {
42
+
$commit = idx($commit_map, $modified_at);
43
+
if ($commit) {
44
+
$commits[$path] = $commit;
46
45
}
47
46
}
47
+
}
48
48
49
-
$commits[$path] = $commit;
49
+
$phids = array();
50
+
foreach ($commits as $commit) {
51
+
$data = $commit->getCommitData();
52
+
$phids[] = $data->getCommitDetail('authorPHID');
53
+
$phids[] = $data->getCommitDetail('committerPHID');
50
54
}
51
-
52
-
$phids = array_keys($phids);
55
+
$phids = array_filter($phids);
53
56
$handles = $this->loadViewerHandles($phids);
54
57
55
58
$branch = $drequest->loadBranch();
56
-
if ($branch) {
59
+
if ($branch && $commits) {
57
60
$lint_query = id(new DiffusionLintCountQuery())
58
61
->withBranchIDs(array($branch->getID()))
59
62
->withPaths(array_keys($commits));
+3
src/applications/diffusion/query/DiffusionCommitQuery.php
+3
src/applications/diffusion/query/DiffusionCommitQuery.php
···
250
250
$data = mpull($data, null, 'getCommitID');
251
251
foreach ($commits as $commit) {
252
252
$commit_data = idx($data, $commit->getID());
253
+
if (!$commit_data) {
254
+
$commit_data = new PhabricatorRepositoryCommitData();
255
+
}
253
256
$commit->attachCommitData($commit_data);
254
257
}
255
258
}