@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Rewrite CommitQuery to use UNION for performance
Summary:
Ref T12680. See PHI167. See that task for discussion.
Rewrite `DiffusionCommitQuery` to work more like `DifferentialRevisionQuery`, and use a UNION to find "all revisions you need to audit OR respond to".
I tried to get this working a little more cleanly than RevisionQuery does, and can probably simplify that now.
Test Plan: Poked at the UI locally without hitting any apparent issues, but my local data is pretty garbage at this point. I'll take a look at how the query plans work on `secure`.
Reviewers: amckinley
Reviewed By: amckinley
Maniphest Tasks: T12680
Differential Revision: https://secure.phabricator.com/D18722
+85
-33
+58
-25
src/applications/diffusion/query/DiffusionCommitQuery.php
+58
-25
src/applications/diffusion/query/DiffusionCommitQuery.php
···
177
177
}
178
178
179
179
protected function loadPage() {
180
-
return $this->loadStandardPage($this->newResultObject());
180
+
$table = $this->newResultObject();
181
+
$conn = $table->establishConnection('r');
182
+
183
+
$subqueries = array();
184
+
if ($this->responsiblePHIDs) {
185
+
$base_authors = $this->authorPHIDs;
186
+
$base_auditors = $this->auditorPHIDs;
187
+
188
+
$responsible_phids = $this->responsiblePHIDs;
189
+
if ($base_authors) {
190
+
$all_authors = array_merge($base_authors, $responsible_phids);
191
+
} else {
192
+
$all_authors = $responsible_phids;
193
+
}
194
+
195
+
if ($base_auditors) {
196
+
$all_auditors = array_merge($base_auditors, $responsible_phids);
197
+
} else {
198
+
$all_auditors = $responsible_phids;
199
+
}
200
+
201
+
$this->authorPHIDs = $all_authors;
202
+
$this->auditorPHIDs = $base_auditors;
203
+
$subqueries[] = $this->buildStandardPageQuery(
204
+
$conn,
205
+
$table->getTableName());
206
+
207
+
$this->authorPHIDs = $base_authors;
208
+
$this->auditorPHIDs = $all_auditors;
209
+
$subqueries[] = $this->buildStandardPageQuery(
210
+
$conn,
211
+
$table->getTableName());
212
+
} else {
213
+
$subqueries[] = $this->buildStandardPageQuery(
214
+
$conn,
215
+
$table->getTableName());
216
+
}
217
+
218
+
if (count($subqueries) > 1) {
219
+
foreach ($subqueries as $key => $subquery) {
220
+
$subqueries[$key] = '('.$subquery.')';
221
+
}
222
+
223
+
$query = qsprintf(
224
+
$conn,
225
+
'%Q %Q %Q',
226
+
implode(' UNION DISTINCT ', $subqueries),
227
+
$this->buildOrderClause($conn, true),
228
+
$this->buildLimitClause($conn));
229
+
} else {
230
+
$query = head($subqueries);
231
+
}
232
+
233
+
$rows = queryfx_all($conn, '%Q', $query);
234
+
$rows = $this->didLoadRawRows($rows);
235
+
236
+
return $table->loadAllFromArray($rows);
181
237
}
182
238
183
239
protected function willFilterPage(array $commits) {
···
487
543
$this->auditorPHIDs);
488
544
}
489
545
490
-
if ($this->responsiblePHIDs !== null) {
491
-
$where[] = qsprintf(
492
-
$conn,
493
-
'(audit.auditorPHID IN (%Ls) OR commit.authorPHID IN (%Ls))',
494
-
$this->responsiblePHIDs,
495
-
$this->responsiblePHIDs);
496
-
}
497
-
498
546
if ($this->statuses !== null) {
499
547
$where[] = qsprintf(
500
548
$conn,
501
-
'commit.auditStatus IN (%Ls)',
549
+
'commit.auditStatus IN (%Ld)',
502
550
$this->statuses);
503
551
}
504
552
···
541
589
return ($this->auditIDs || $this->auditorPHIDs);
542
590
}
543
591
544
-
private function shouldJoinAudit() {
545
-
return (bool)$this->responsiblePHIDs;
546
-
}
547
-
548
592
private function shouldJoinOwners() {
549
593
return (bool)$this->packagePHIDs;
550
594
}
···
560
604
$audit_request->getTableName());
561
605
}
562
606
563
-
if ($this->shouldJoinAudit()) {
564
-
$join[] = qsprintf(
565
-
$conn,
566
-
'LEFT JOIN %T audit ON commit.phid = audit.commitPHID',
567
-
$audit_request->getTableName());
568
-
}
569
-
570
607
if ($this->shouldJoinOwners()) {
571
608
$join[] = qsprintf(
572
609
$conn,
···
581
618
582
619
protected function shouldGroupQueryResultRows() {
583
620
if ($this->shouldJoinAuditor()) {
584
-
return true;
585
-
}
586
-
587
-
if ($this->shouldJoinAudit()) {
588
621
return true;
589
622
}
590
623
+27
-8
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
+27
-8
src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php
···
111
111
AphrontDatabaseConnection $conn,
112
112
$table_name) {
113
113
114
-
$rows = queryfx_all(
114
+
$query = $this->buildStandardPageQuery($conn, $table_name);
115
+
116
+
$rows = queryfx_all($conn, '%Q', $query);
117
+
$rows = $this->didLoadRawRows($rows);
118
+
119
+
return $rows;
120
+
}
121
+
122
+
protected function buildStandardPageQuery(
123
+
AphrontDatabaseConnection $conn,
124
+
$table_name) {
125
+
126
+
return qsprintf(
115
127
$conn,
116
128
'%Q FROM %T %Q %Q %Q %Q %Q %Q %Q',
117
129
$this->buildSelectClause($conn),
···
123
135
$this->buildHavingClause($conn),
124
136
$this->buildOrderClause($conn),
125
137
$this->buildLimitClause($conn));
126
-
127
-
$rows = $this->didLoadRawRows($rows);
128
-
129
-
return $rows;
130
138
}
131
139
132
140
protected function didLoadRawRows(array $rows) {
···
1032
1040
/**
1033
1041
* @task order
1034
1042
*/
1035
-
final protected function buildOrderClause(AphrontDatabaseConnection $conn) {
1043
+
final protected function buildOrderClause(
1044
+
AphrontDatabaseConnection $conn,
1045
+
$for_union = false) {
1046
+
1036
1047
$orderable = $this->getOrderableColumns();
1037
1048
$vector = $this->getOrderVector();
1038
1049
···
1045
1056
$parts[] = $part;
1046
1057
}
1047
1058
1048
-
return $this->formatOrderClause($conn, $parts);
1059
+
return $this->formatOrderClause($conn, $parts, $for_union);
1049
1060
}
1050
1061
1051
1062
···
1054
1065
*/
1055
1066
protected function formatOrderClause(
1056
1067
AphrontDatabaseConnection $conn,
1057
-
array $parts) {
1068
+
array $parts,
1069
+
$for_union = false) {
1058
1070
1059
1071
$is_query_reversed = false;
1060
1072
if ($this->getBeforeID()) {
···
1075
1087
}
1076
1088
1077
1089
$table = idx($part, 'table');
1090
+
1091
+
// When we're building an ORDER BY clause for a sequence of UNION
1092
+
// statements, we can't refer to tables from the subqueries.
1093
+
if ($for_union) {
1094
+
$table = null;
1095
+
}
1096
+
1078
1097
$column = $part['column'];
1079
1098
1080
1099
if ($table !== null) {