@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Truncate object fields in Herald transcripts
Summary:
A few users have hit cases where Herald transcripts of large commits exceed the MySQL packet limit, because one of the fields in the transcript is an enomrous textual diff.
There's no value in saving these huge amounts of data. Transcripts are useful for understanding the action of Herald rules, but can be reconstructed later. Instead of saving all of the data, limit each field to 4KB of data.
For strings, we just truncate at 4KB. For arrays, we truncate after 4KB of values.
Test Plan: Ran unit tests. Artificially decreased limit and ran transcripts, saw them truncate properly.
Reviewers: btrahan
Reviewed By: btrahan
CC: frgtn, aran
Differential Revision: https://secure.phabricator.com/D7783
+79
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
779
779
'HeraldTranscriptController' => 'applications/herald/controller/HeraldTranscriptController.php',
780
780
'HeraldTranscriptListController' => 'applications/herald/controller/HeraldTranscriptListController.php',
781
781
'HeraldTranscriptQuery' => 'applications/herald/query/HeraldTranscriptQuery.php',
782
+
'HeraldTranscriptTestCase' => 'applications/herald/storage/__tests__/HeraldTranscriptTestCase.php',
782
783
'Javelin' => 'infrastructure/javelin/Javelin.php',
783
784
'JavelinReactorExample' => 'applications/uiexample/examples/JavelinReactorExample.php',
784
785
'JavelinUIExample' => 'applications/uiexample/examples/JavelinUIExample.php',
···
3211
3212
'HeraldTranscriptController' => 'HeraldController',
3212
3213
'HeraldTranscriptListController' => 'HeraldController',
3213
3214
'HeraldTranscriptQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
3215
+
'HeraldTranscriptTestCase' => 'PhabricatorTestCase',
3214
3216
'JavelinReactorExample' => 'PhabricatorUIExample',
3215
3217
'JavelinUIExample' => 'PhabricatorUIExample',
3216
3218
'JavelinViewExample' => 'PhabricatorUIExample',
+47
src/applications/herald/storage/__tests__/HeraldTranscriptTestCase.php
+47
src/applications/herald/storage/__tests__/HeraldTranscriptTestCase.php
···
1
+
<?php
2
+
3
+
final class HeraldTranscriptTestCase extends PhabricatorTestCase {
4
+
5
+
public function testTranscriptTruncation() {
6
+
$long_string = str_repeat('x', 1024 * 1024);
7
+
$short_string = str_repeat('x', 4096)."\n<...>";
8
+
9
+
$long_array = array(
10
+
'a' => $long_string,
11
+
'b' => $long_string,
12
+
);
13
+
14
+
$mixed_array = array(
15
+
'a' => 'abc',
16
+
'b' => 'def',
17
+
'c' => $long_string,
18
+
);
19
+
20
+
$fields = array(
21
+
'ls' => $long_string,
22
+
'la' => $long_array,
23
+
'ma' => $mixed_array,
24
+
);
25
+
26
+
$truncated_fields = id(new HeraldObjectTranscript())
27
+
->setFields($fields)
28
+
->getFields();
29
+
30
+
$this->assertEqual($short_string, $truncated_fields['ls']);
31
+
32
+
$this->assertEqual(
33
+
array('a', '<...>'),
34
+
array_keys($truncated_fields['la']));
35
+
$this->assertEqual(
36
+
$short_string.'!<...>',
37
+
implode('!', $truncated_fields['la']));
38
+
39
+
$this->assertEqual(
40
+
array('a', 'b', 'c'),
41
+
array_keys($truncated_fields['ma']));
42
+
$this->assertEqual(
43
+
'abc!def!'.substr($short_string, 6),
44
+
implode('!', $truncated_fields['ma']));
45
+
46
+
}
47
+
}
+30
src/applications/herald/storage/transcript/HeraldObjectTranscript.php
+30
src/applications/herald/storage/transcript/HeraldObjectTranscript.php
···
35
35
}
36
36
37
37
public function setFields(array $fields) {
38
+
foreach ($fields as $key => $value) {
39
+
$fields[$key] = self::truncateValue($value, 4096);
40
+
}
41
+
38
42
$this->fields = $fields;
39
43
return $this;
40
44
}
···
42
46
public function getFields() {
43
47
return $this->fields;
44
48
}
49
+
50
+
private static function truncateValue($value, $length) {
51
+
if (is_string($value)) {
52
+
if (strlen($value) <= $length) {
53
+
return $value;
54
+
} else {
55
+
// NOTE: phutil_utf8_shorten() has huge runtime for giant strings.
56
+
return phutil_utf8ize(substr($value, 0, $length)."\n<...>");
57
+
}
58
+
} else if (is_array($value)) {
59
+
foreach ($value as $key => $v) {
60
+
if ($length <= 0) {
61
+
$value['<...>'] = '<...>';
62
+
unset($value[$key]);
63
+
} else {
64
+
$v = self::truncateValue($v, $length);
65
+
$length -= strlen($v);
66
+
$value[$key] = $v;
67
+
}
68
+
}
69
+
return $value;
70
+
} else {
71
+
return $value;
72
+
}
73
+
}
74
+
45
75
}