@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Limit maximum size of Owners package queries
Summary:
Currently, a change may affect a very large number of paths. When we run the OwnersWorker on it, we'll execute a query which looks up packages for the paths. This may exceed "max_allowed_packet". Instead, break the list of paths into smaller chunks.
This is mostly to unblock r4nt / llvm, I'm going to add a more finessed approach to array_chunk() shortly.
Test Plan: Ran `reparse.php --owners` on a revision which affected packages, verified results are the same before and after the change. Set chunk size to 1, verified query results aggregated properly.
Reviewers: btrahan, jungejason, nh
Reviewed By: jungejason
CC: aran
Differential Revision: https://secure.phabricator.com/D2943
+35
-24
+35
-24
src/applications/owners/storage/PhabricatorOwnersPackage.php
+35
-24
src/applications/owners/storage/PhabricatorOwnersPackage.php
···
142
142
$path = new PhabricatorOwnersPath();
143
143
$conn = $package->establishConnection('r');
144
144
145
-
$repository_clause = qsprintf($conn, 'AND p.repositoryPHID = %s',
145
+
$repository_clause = qsprintf(
146
+
$conn,
147
+
'AND p.repositoryPHID = %s',
146
148
$repository->getPHID());
147
149
148
-
$limit_clause = '';
149
-
if (!empty($limit)) {
150
-
$limit_clause = qsprintf($conn, 'LIMIT %d', $limit);
151
-
}
150
+
// NOTE: The list of $paths may be very large if we're coming from
151
+
// the OwnersWorker and processing, e.g., an SVN commit which created a new
152
+
// branch. Break it apart so that it will fit within 'max_allowed_packet',
153
+
// and then merge results in PHP.
152
154
153
-
$data = queryfx_all(
154
-
$conn,
155
-
'SELECT pkg.id FROM %T pkg JOIN %T p ON p.packageID = pkg.id
156
-
WHERE p.path IN (%Ls) %Q ORDER BY LENGTH(p.path) DESC %Q',
157
-
$package->getTableName(),
158
-
$path->getTableName(),
159
-
$paths,
160
-
$repository_clause,
161
-
$limit_clause);
155
+
$ids = array();
156
+
foreach (array_chunk($paths, 128) as $chunk) {
157
+
$rows = queryfx_all(
158
+
$conn,
159
+
'SELECT pkg.id id, LENGTH(p.path) len
160
+
FROM %T pkg JOIN %T p ON p.packageID = pkg.id
161
+
WHERE p.path IN (%Ls) %Q',
162
+
$package->getTableName(),
163
+
$path->getTableName(),
164
+
$chunk,
165
+
$repository_clause);
162
166
163
-
$ids = ipull($data, 'id');
167
+
foreach ($rows as $row) {
168
+
$id = (int)$row['id'];
169
+
$len = (int)$row['len'];
170
+
if (isset($ids[$id])) {
171
+
$ids[$id] = max($len, $ids[$id]);
172
+
} else {
173
+
$ids[$id] = $len;
174
+
}
175
+
}
176
+
}
164
177
165
-
if (empty($ids)) {
178
+
if (!$ids) {
166
179
return array();
167
180
}
168
181
169
-
$order = array();
170
-
foreach ($ids as $id) {
171
-
if (empty($order[$id])) {
172
-
$order[$id] = true;
173
-
}
182
+
arsort($ids);
183
+
if ($limit) {
184
+
$ids = array_slice($ids, 0, $limit, $preserve_keys = true);
174
185
}
186
+
$ids = array_keys($ids);
175
187
176
-
$packages = $package->loadAllWhere('id in (%Ld)', array_keys($order));
177
-
178
-
$packages = array_select_keys($packages, array_keys($order));
188
+
$packages = $package->loadAllWhere('id in (%Ld)', array_keys($ids));
189
+
$packages = array_select_keys($packages, array_keys($ids));
179
190
180
191
return $packages;
181
192
}