@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Cut Herald rules off at 1GB of diff text
Summary:
Ref T4276. When a change is larger than 2GB, PHP can not read the entire change into a string, so Herald can not process it.
Additionally, we already have a time limit for practical reasons, but it's huge (probably incorrectly). To deal with these things:
- Add an optional byte limit to `diffusion.rawdiffquery`.
- Make the query with a 1GB limit.
- Reduce the diff timeout from 15 hours to 15 minutes.
- Add a "Changeset is enormous" field. This field is true for changes which are too large to process.
This generally makes behaviors more sane:
- We'll always make progress in Herald in a reasonable amount of time.
- Installs can write global rules to handle (or reject) these types of changes.
Test Plan: Set limit to 25 bytes instead of 1GB and ran test console on various changes.
Reviewers: btrahan
Reviewed By: btrahan
CC: aran
Maniphest Tasks: T4276
Differential Revision: https://secure.phabricator.com/D7885
+60
-19
+13
-3
src/applications/diffusion/conduit/ConduitAPI_diffusion_rawdiffquery_Method.php
+13
-3
src/applications/diffusion/conduit/ConduitAPI_diffusion_rawdiffquery_Method.php
···
21
21
'commit' => 'required string',
22
22
'path' => 'optional string',
23
23
'timeout' => 'optional int',
24
+
'byteLimit' => 'optional int',
24
25
'linesOfContext' => 'optional int',
25
26
'againstCommit' => 'optional string',
26
27
);
···
28
29
29
30
protected function getResult(ConduitAPIRequest $request) {
30
31
$drequest = $this->getDiffusionRequest();
31
-
$timeout = $request->getValue('timeout');
32
-
$lines_of_context = $request->getValue('linesOfContext');
33
-
$against_commit = $request->getValue('againstCommit');
34
32
35
33
$raw_query = DiffusionRawDiffQuery::newFromDiffusionRequest($drequest);
34
+
35
+
$timeout = $request->getValue('timeout');
36
36
if ($timeout !== null) {
37
37
$raw_query->setTimeout($timeout);
38
38
}
39
+
40
+
$lines_of_context = $request->getValue('linesOfContext');
39
41
if ($lines_of_context !== null) {
40
42
$raw_query->setLinesOfContext($lines_of_context);
41
43
}
44
+
45
+
$against_commit = $request->getValue('againstCommit');
42
46
if ($against_commit !== null) {
43
47
$raw_query->setAgainstCommit($against_commit);
44
48
}
49
+
50
+
$byte_limit = $request->getValue('byteLimit');
51
+
if ($byte_limit !== null) {
52
+
$raw_query->setByteLimit($byte_limit);
53
+
}
54
+
45
55
return $raw_query->loadRawDiff();
46
56
}
47
57
}
+2
-6
src/applications/diffusion/query/rawdiff/DiffusionGitRawDiffQuery.php
+2
-6
src/applications/diffusion/query/rawdiff/DiffusionGitRawDiffQuery.php
···
34
34
$commit,
35
35
$path);
36
36
37
-
if ($this->getTimeout()) {
38
-
$future->setTimeout($this->getTimeout());
39
-
}
37
+
$this->configureFuture($future);
40
38
41
39
try {
42
40
list($raw_diff) = $future->resolvex();
···
61
59
$commit,
62
60
$drequest->getPath());
63
61
64
-
if ($this->getTimeout()) {
65
-
$future->setTimeout($this->getTimeout());
66
-
}
62
+
$this->configureFuture($future);
67
63
68
64
list($raw_diff) = $future->resolvex();
69
65
}
+1
-4
src/applications/diffusion/query/rawdiff/DiffusionMercurialRawDiffQuery.php
+1
-4
src/applications/diffusion/query/rawdiff/DiffusionMercurialRawDiffQuery.php
···
6
6
return $this->executeRawDiffCommand();
7
7
}
8
8
9
-
10
9
protected function executeRawDiffCommand() {
11
10
$drequest = $this->getRequest();
12
11
$repository = $drequest->getRepository();
···
31
30
$commit,
32
31
$path);
33
32
34
-
if ($this->getTimeout()) {
35
-
$future->setTimeout($this->getTimeout());
36
-
}
33
+
$this->configureFuture($future);
37
34
38
35
list($raw_diff) = $future->resolvex();
39
36
+21
src/applications/diffusion/query/rawdiff/DiffusionRawDiffQuery.php
+21
src/applications/diffusion/query/rawdiff/DiffusionRawDiffQuery.php
···
6
6
private $timeout;
7
7
private $linesOfContext = 65535;
8
8
private $againstCommit;
9
+
private $byteLimit;
9
10
10
11
final public static function newFromDiffusionRequest(
11
12
DiffusionRequest $request) {
···
25
26
return $this->timeout;
26
27
}
27
28
29
+
public function setByteLimit($byte_limit) {
30
+
$this->byteLimit = $byte_limit;
31
+
return $this;
32
+
}
33
+
34
+
public function getByteLimit() {
35
+
return $this->byteLimit;
36
+
}
37
+
28
38
final public function setLinesOfContext($lines_of_context) {
29
39
$this->linesOfContext = $lines_of_context;
30
40
return $this;
···
41
51
42
52
final public function getAgainstCommit() {
43
53
return $this->againstCommit;
54
+
}
55
+
56
+
protected function configureFuture(ExecFuture $future) {
57
+
if ($this->getTimeout()) {
58
+
$future->setTimeout($this->getTimeout());
59
+
}
60
+
61
+
if ($this->getByteLimit()) {
62
+
$future->setStdoutSizeLimit($this->getByteLimit());
63
+
$future->setStderrSizeLimit($this->getByteLimit());
64
+
}
44
65
}
45
66
46
67
}
+1
-3
src/applications/diffusion/query/rawdiff/DiffusionSvnRawDiffQuery.php
+1
-3
src/applications/diffusion/query/rawdiff/DiffusionSvnRawDiffQuery.php
+3
src/applications/herald/adapter/HeraldAdapter.php
+3
src/applications/herald/adapter/HeraldAdapter.php
···
18
18
const FIELD_DIFF_CONTENT = 'diff-content';
19
19
const FIELD_DIFF_ADDED_CONTENT = 'diff-added-content';
20
20
const FIELD_DIFF_REMOVED_CONTENT = 'diff-removed-content';
21
+
const FIELD_DIFF_ENORMOUS = 'diff-enormous';
21
22
const FIELD_REPOSITORY = 'repository';
22
23
const FIELD_REPOSITORY_PROJECTS = 'repository-projects';
23
24
const FIELD_RULE = 'rule';
···
193
194
self::FIELD_DIFF_CONTENT => pht('Any changed file content'),
194
195
self::FIELD_DIFF_ADDED_CONTENT => pht('Any added file content'),
195
196
self::FIELD_DIFF_REMOVED_CONTENT => pht('Any removed file content'),
197
+
self::FIELD_DIFF_ENORMOUS => pht('Change is enormous'),
196
198
self::FIELD_REPOSITORY => pht('Repository'),
197
199
self::FIELD_REPOSITORY_PROJECTS => pht('Repository\'s projects'),
198
200
self::FIELD_RULE => pht('Another Herald rule'),
···
342
344
self::CONDITION_NOT_EXISTS,
343
345
);
344
346
case self::FIELD_IS_MERGE_COMMIT:
347
+
case self::FIELD_DIFF_ENORMOUS:
345
348
return array(
346
349
self::CONDITION_IS_TRUE,
347
350
self::CONDITION_IS_FALSE,
+18
-2
src/applications/herald/adapter/HeraldCommitAdapter.php
+18
-2
src/applications/herald/adapter/HeraldCommitAdapter.php
···
99
99
self::FIELD_DIFF_CONTENT,
100
100
self::FIELD_DIFF_ADDED_CONTENT,
101
101
self::FIELD_DIFF_REMOVED_CONTENT,
102
+
self::FIELD_DIFF_ENORMOUS,
102
103
self::FIELD_RULE,
103
104
self::FIELD_AFFECTED_PACKAGE,
104
105
self::FIELD_AFFECTED_PACKAGE_OWNER,
···
277
278
'commit' => $this->commit->getCommitIdentifier(),
278
279
));
279
280
281
+
$byte_limit = (1024 * 1024 * 1024); // 1GB
282
+
280
283
$raw = DiffusionQuery::callConduitWithDiffusionRequest(
281
284
PhabricatorUser::getOmnipotentUser(),
282
285
$drequest,
283
286
'diffusion.rawdiffquery',
284
287
array(
285
288
'commit' => $this->commit->getCommitIdentifier(),
286
-
'timeout' => 60 * 60 * 15,
287
-
'linesOfContext' => 0));
289
+
'timeout' => (60 * 15), // 15 minutes
290
+
'byteLimit' => $byte_limit,
291
+
'linesOfContext' => 0,
292
+
));
293
+
294
+
if (strlen($raw) >= $byte_limit) {
295
+
throw new Exception(
296
+
pht(
297
+
'The raw text of this change is enormous (larger than %d bytes). '.
298
+
'Herald can not process it.',
299
+
$byte_limit));
300
+
}
288
301
289
302
$parser = new ArcanistDiffParser();
290
303
$changes = $parser->parseDiff($raw);
···
360
373
return $this->getDiffContent('+');
361
374
case self::FIELD_DIFF_REMOVED_CONTENT:
362
375
return $this->getDiffContent('-');
376
+
case self::FIELD_DIFF_ENORMOUS:
377
+
$this->getDiffContent('*');
378
+
return ($this->commitDiff instanceof Exception);
363
379
case self::FIELD_AFFECTED_PACKAGE:
364
380
$packages = $this->loadAffectedPackages();
365
381
return mpull($packages, 'getPHID');
+1
-1
src/applications/herald/storage/HeraldRule.php
+1
-1
src/applications/herald/storage/HeraldRule.php