@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add GROUP BY to commit query
Summary:
Ref T4715. Some minor stuff I caught locally while poking around:
- Since we don't `GROUP BY`, we can still get duplicate commits. These get silently de-duplicated by `loadAllFromArray()` because that returns an array keyed by `id`, but we fetch too much data and this can cause us to execute too many queries to fill pages. Instead, `GROUP BY` if we joined the audit table.
- After adding `GROUP BY`, getting the audit IDs out of the query is no longer reliable. Instead, query audits by the commit PHIDs. This is approximately equiavlent.
- Since we always `JOIN`, we currently never return commits that don't have any audits. If we don't know that all results will have an audit, just `LEFT JOIN`.
- Add some `!== null` to catch the `withIDs(array())` issue that we hit with Khan Academy a little while ago.
Test Plan:
- Verified that "All Commits" shows commits with no audits of any kind.
- Verified that the raw data comes out of the query without duplicates.
Reviewers: btrahan
Reviewed By: btrahan
Subscribers: epriestley
Maniphest Tasks: T5433, T4715
Differential Revision: https://secure.phabricator.com/D8879
+61
-36
+12
src/applications/audit/conduit/ConduitAPI_audit_query_Method.php
+12
src/applications/audit/conduit/ConduitAPI_audit_query_Method.php
···
51
51
DiffusionCommitQuery::AUDIT_STATUS_ANY);
52
52
$query->withAuditStatus($status);
53
53
54
+
// NOTE: These affect the number of commits identified, which is sort of
55
+
// reasonable but means the method may return an arbitrary number of
56
+
// actual audit requests.
54
57
$query->setOffset($request->getValue('offset', 0));
55
58
$query->setLimit($request->getValue('limit', 100));
56
59
57
60
$commits = $query->execute();
58
61
62
+
$auditor_map = array_fuse($auditor_phids);
63
+
59
64
$results = array();
60
65
foreach ($commits as $commit) {
61
66
$requests = $commit->getAudits();
62
67
foreach ($requests as $request) {
68
+
69
+
// If this audit isn't triggered for one of the requested PHIDs,
70
+
// skip it.
71
+
if ($auditor_map && empty($auditor_map[$request->getAuditorPHID()])) {
72
+
continue;
73
+
}
74
+
63
75
$results[] = array(
64
76
'id' => $request->getID(),
65
77
'commitPHID' => $request->getCommitPHID(),
+20
-6
src/applications/audit/management/PhabricatorAuditManagementDeleteWorkflow.php
+20
-6
src/applications/audit/management/PhabricatorAuditManagementDeleteWorkflow.php
···
85
85
$query->withAuditStatus($status);
86
86
}
87
87
88
+
$id_map = array();
88
89
if ($ids) {
90
+
$id_map = array_fuse($ids);
89
91
$query->withAuditIDs($ids);
90
92
}
91
93
···
93
95
$query->withRepositoryIDs(mpull($repos, 'getID'));
94
96
}
95
97
98
+
$auditor_map = array();
96
99
if ($users) {
97
-
$query->withAuditorPHIDs(mpull($users, 'getPHID'));
100
+
$auditor_map = array_fuse(mpull($users, 'getPHID'));
101
+
$query->withAuditorPHIDs($auditor_map);
98
102
}
99
103
100
104
if ($commits) {
···
105
109
$commits = mpull($commits, null, 'getPHID');
106
110
$audits = array();
107
111
foreach ($commits as $commit) {
108
-
$curr_audits = $commit->getAudits();
109
-
foreach ($audits as $key => $audit) {
112
+
$commit_audits = $commit->getAudits();
113
+
foreach ($commit_audits as $key => $audit) {
114
+
if ($id_map && empty($id_map[$audit->getID()])) {
115
+
unset($commit_audits[$key]);
116
+
continue;
117
+
}
118
+
119
+
if ($auditor_map && empty($auditor_map[$audit->getAuditorPHID()])) {
120
+
unset($commit_audits[$key]);
121
+
continue;
122
+
}
123
+
110
124
if ($min_date && $commit->getEpoch() < $min_date) {
111
-
unset($audits[$key]);
125
+
unset($commit_audits[$key]);
112
126
continue;
113
127
}
114
128
115
129
if ($max_date && $commit->getEpoch() > $max_date) {
116
-
unset($audits[$key]);
130
+
unset($commit_audits[$key]);
117
131
continue;
118
132
}
119
133
}
120
-
$audits[] = $curr_audits;
134
+
$audits[] = $commit_audits;
121
135
}
122
136
$audits = array_mergev($audits);
123
137
+29
-30
src/applications/diffusion/query/DiffusionCommitQuery.php
+29
-30
src/applications/diffusion/query/DiffusionCommitQuery.php
···
19
19
const AUDIT_STATUS_ANY = 'audit-status-any';
20
20
const AUDIT_STATUS_OPEN = 'audit-status-open';
21
21
const AUDIT_STATUS_CONCERN = 'audit-status-concern';
22
-
private $loadAuditIds;
23
22
24
23
private $needCommitData;
25
24
···
94
93
* rows must always have it.
95
94
*/
96
95
private function shouldJoinAudits() {
97
-
return
98
-
$this->needAuditRequests ||
99
-
$this->auditStatus ||
100
-
$this->rowsMustHaveAudits();
96
+
return $this->auditStatus ||
97
+
$this->rowsMustHaveAudits();
101
98
}
102
99
103
100
···
156
153
157
154
$data = queryfx_all(
158
155
$conn_r,
159
-
'SELECT commit.* %Q FROM %T commit %Q %Q %Q %Q',
160
-
$this->buildAuditSelect($conn_r),
156
+
'SELECT commit.* FROM %T commit %Q %Q %Q %Q %Q',
161
157
$table->getTableName(),
162
158
$this->buildJoinClause($conn_r),
163
159
$this->buildWhereClause($conn_r),
160
+
$this->buildGroupClause($conn_r),
164
161
$this->buildOrderClause($conn_r),
165
162
$this->buildLimitClause($conn_r));
166
-
167
-
if ($this->shouldJoinAudits()) {
168
-
$this->loadAuditIds = ipull($data, 'audit_id');
169
-
}
170
163
171
164
return $table->loadAllFromArray($data);
172
165
}
173
166
174
-
private function buildAuditSelect($conn_r) {
175
-
if ($this->shouldJoinAudits()) {
176
-
return qsprintf(
177
-
$conn_r,
178
-
', audit.id as audit_id');
179
-
}
180
-
181
-
return '';
182
-
}
183
-
184
167
protected function willFilterPage(array $commits) {
185
168
$repository_ids = mpull($commits, 'getRepositoryID', 'getRepositoryID');
186
169
$repos = id(new PhabricatorRepositoryQuery())
···
230
213
$result[$identifier] = head($matching_commits);
231
214
} else {
232
215
// This reference is ambiguous (it matches more than one commit) so
233
-
// don't link it
216
+
// don't link it.
234
217
unset($result[$identifier]);
235
218
}
236
219
}
···
257
240
}
258
241
}
259
242
260
-
if ($this->shouldJoinAudits()) {
261
-
$load_ids = array_filter($this->loadAuditIds);
262
-
if ($load_ids) {
263
-
$requests = id(new PhabricatorRepositoryAuditRequest())
264
-
->loadAllWhere('id IN (%Ld)', $this->loadAuditIds);
265
-
} else {
266
-
$requests = array();
267
-
}
243
+
// TODO: This should just be `needAuditRequests`, not `shouldJoinAudits()`,
244
+
// but leave that for a future diff.
245
+
246
+
if ($this->needAuditRequests || $this->shouldJoinAudits()) {
247
+
$requests = id(new PhabricatorRepositoryAuditRequest())->loadAllWhere(
248
+
'commitPHID IN (%Ls)',
249
+
mpull($commits, 'getPHID'));
268
250
269
251
$requests = mgroup($requests, 'getCommitPHID');
270
252
foreach ($commits as $commit) {
···
503
485
504
486
if ($joins) {
505
487
return implode(' ', $joins);
488
+
} else {
489
+
return '';
490
+
}
491
+
}
492
+
493
+
private function buildGroupClause(AphrontDatabaseConnection $conn_r) {
494
+
$should_group = $this->shouldJoinAudits();
495
+
496
+
// TODO: Currently, the audit table is missing a unique key, so we may
497
+
// require a GROUP BY if we perform this join. See T1768. This can be
498
+
// removed once the table has the key.
499
+
if ($this->auditAwaitingUser) {
500
+
$should_group = true;
501
+
}
502
+
503
+
if ($should_group) {
504
+
return 'GROUP BY commit.id';
506
505
} else {
507
506
return '';
508
507
}