@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Use PhutilRope as a buffer in Harbormaster BuildLogs
Summary:
Ref T10457. Currently, every `append()` call necessarily generates queries, and these queries are slightly inefficient if a large block of data is appended to a partial log (they do about twice as much work as they technically need to).
Use `PhutilRope` to buffer `append()` input so the logic is a little cleaner and we could add a rule like "flush logs no more than once every 500ms" later.
Test Plan:
- Ran a build, saw logs.
- Set chunk size very small, ran build, saw logs, verified small logs in database.
{F1137115}
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T10457
Differential Revision: https://secure.phabricator.com/D15375
+75
-57
+1
-1
src/applications/harbormaster/storage/HarbormasterSchemaSpec.php
+1
-1
src/applications/harbormaster/storage/HarbormasterSchemaSpec.php
+74
-56
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+74
-56
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
···
11
11
protected $live;
12
12
13
13
private $buildTarget = self::ATTACHABLE;
14
+
private $rope;
15
+
private $isOpen;
14
16
15
17
const CHUNK_BYTE_LIMIT = 102400;
18
+
const CHUNK_TABLE = 'harbormaster_buildlogchunk';
16
19
17
20
/**
18
21
* The log is encoded as plain text.
19
22
*/
20
23
const ENCODING_TEXT = 'text';
21
24
25
+
public function __construct() {
26
+
$this->rope = new PhutilRope();
27
+
}
28
+
22
29
public function __destruct() {
23
-
if ($this->getLive()) {
30
+
if ($this->isOpen) {
24
31
$this->closeBuildLog();
25
32
}
26
33
}
···
35
42
}
36
43
37
44
public function openBuildLog() {
38
-
if ($this->getLive()) {
45
+
if ($this->isOpen) {
39
46
throw new Exception(pht('This build log is already open!'));
40
47
}
48
+
49
+
$this->isOpen = true;
41
50
42
51
return $this
43
52
->setLive(1)
···
45
54
}
46
55
47
56
public function closeBuildLog() {
48
-
if (!$this->getLive()) {
57
+
if (!$this->isOpen) {
49
58
throw new Exception(pht('This build log is not open!'));
50
59
}
51
60
···
108
117
}
109
118
110
119
$content = (string)$content;
111
-
if (!strlen($content)) {
112
-
return;
113
-
}
120
+
121
+
$this->rope->append($content);
122
+
$this->flush();
123
+
}
124
+
125
+
private function flush() {
114
126
115
-
// If the length of the content is greater than the chunk size limit,
116
-
// then we can never fit the content in a single record. We need to
117
-
// split our content out and call append on it for as many parts as there
118
-
// are to the content.
119
-
if (strlen($content) > self::CHUNK_BYTE_LIMIT) {
120
-
$current = $content;
121
-
while (strlen($current) > self::CHUNK_BYTE_LIMIT) {
122
-
$part = substr($current, 0, self::CHUNK_BYTE_LIMIT);
123
-
$current = substr($current, self::CHUNK_BYTE_LIMIT);
124
-
$this->append($part);
127
+
// TODO: Maybe don't flush more than a couple of times per second. If a
128
+
// caller writes a single character over and over again, we'll currently
129
+
// spend a lot of time flushing that.
130
+
131
+
$chunk_table = self::CHUNK_TABLE;
132
+
$chunk_limit = self::CHUNK_BYTE_LIMIT;
133
+
$rope = $this->rope;
134
+
135
+
while (true) {
136
+
$length = $rope->getByteLength();
137
+
if (!$length) {
138
+
break;
125
139
}
126
-
$this->append($current);
127
-
return;
128
-
}
129
140
130
-
// Retrieve the size of last chunk from the DB for this log. If the
131
-
// chunk is over 500K, then we need to create a new log entry.
132
-
$conn = $this->establishConnection('w');
133
-
$result = queryfx_all(
134
-
$conn,
135
-
'SELECT id, size, encoding '.
136
-
'FROM harbormaster_buildlogchunk '.
137
-
'WHERE logID = %d '.
138
-
'ORDER BY id DESC '.
139
-
'LIMIT 1',
140
-
$this->getID());
141
-
if (count($result) === 0 ||
142
-
$result[0]['size'] + strlen($content) > self::CHUNK_BYTE_LIMIT ||
143
-
$result[0]['encoding'] !== self::ENCODING_TEXT) {
141
+
$conn_w = $this->establishConnection('w');
142
+
$tail = queryfx_one(
143
+
$conn_w,
144
+
'SELECT id, size, encoding FROM %T WHERE logID = %d
145
+
ORDER BY id DESC LIMIT 1',
146
+
$chunk_table,
147
+
$this->getID());
144
148
145
-
// We must insert a new chunk because the data we are appending
146
-
// won't fit into the existing one, or we don't have any existing
147
-
// chunk data.
148
-
queryfx(
149
-
$conn,
150
-
'INSERT INTO harbormaster_buildlogchunk '.
151
-
'(logID, encoding, size, chunk) '.
152
-
'VALUES '.
153
-
'(%d, %s, %d, %B)',
154
-
$this->getID(),
155
-
self::ENCODING_TEXT,
156
-
strlen($content),
157
-
$content);
158
-
} else {
159
-
// We have a resulting record that we can append our content onto.
160
-
queryfx(
161
-
$conn,
162
-
'UPDATE harbormaster_buildlogchunk '.
163
-
'SET chunk = CONCAT(chunk, %B), size = LENGTH(CONCAT(chunk, %B))'.
164
-
'WHERE id = %d',
165
-
$content,
166
-
$content,
167
-
$result[0]['id']);
149
+
$can_append =
150
+
($tail) &&
151
+
($tail['encoding'] == self::ENCODING_TEXT) &&
152
+
($tail['size'] < $chunk_limit);
153
+
if ($can_append) {
154
+
$append_id = $tail['id'];
155
+
$prefix_size = $tail['size'];
156
+
} else {
157
+
$append_id = null;
158
+
$prefix_size = 0;
159
+
}
160
+
161
+
$data_limit = ($chunk_limit - $prefix_size);
162
+
$append_data = $rope->getPrefixBytes($data_limit);
163
+
$data_size = strlen($append_data);
164
+
165
+
if ($append_id) {
166
+
queryfx(
167
+
$conn_w,
168
+
'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
169
+
$chunk_table,
170
+
$append_data,
171
+
$prefix_size + $data_size,
172
+
$append_id);
173
+
} else {
174
+
queryfx(
175
+
$conn_w,
176
+
'INSERT INTO %T (logID, encoding, size, chunk)
177
+
VALUES (%d, %s, %d, %B)',
178
+
$chunk_table,
179
+
$this->getID(),
180
+
self::ENCODING_TEXT,
181
+
$data_size,
182
+
$append_data);
183
+
}
184
+
185
+
$rope->removeBytesFromHead(strlen($append_data));
168
186
}
169
187
}
170
188