@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Manage log locks on the Log object to prepare for multiple writers
Summary:
Depends on D19134. Ref T13088. Future changes will support API writers, so push the log lock into the Log object.
Allow open/close ("this process is writing to this log") to be separate from live/final ("this log is still generating more data").
Test Plan: Wrote logs with `bin/harbormater write-log` and updated logs with `bin/harbormaster rebuild-log`.
Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam
Maniphest Tasks: T13088
Differential Revision: https://secure.phabricator.com/D19135
+174
-113
+161
-100
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
+161
-100
src/applications/harbormaster/storage/build/HarbormasterBuildLog.php
···
16
16
private $buildTarget = self::ATTACHABLE;
17
17
private $rope;
18
18
private $isOpen;
19
+
private $lock;
19
20
20
21
const CHUNK_BYTE_LIMIT = 1048576;
21
22
···
27
28
if ($this->isOpen) {
28
29
$this->closeBuildLog();
29
30
}
31
+
32
+
if ($this->lock) {
33
+
if ($this->lock->isLocked()) {
34
+
$this->lock->unlock();
35
+
}
36
+
}
30
37
}
31
38
32
39
public static function initializeNewBuildLog(
···
35
42
return id(new HarbormasterBuildLog())
36
43
->setBuildTargetPHID($build_target->getPHID())
37
44
->setDuration(null)
38
-
->setLive(0);
39
-
}
40
-
41
-
public function openBuildLog() {
42
-
if ($this->isOpen) {
43
-
throw new Exception(pht('This build log is already open!'));
44
-
}
45
-
46
-
$this->isOpen = true;
47
-
48
-
return $this
49
-
->setLive(1)
50
-
->save();
51
-
}
52
-
53
-
public function closeBuildLog() {
54
-
if (!$this->isOpen) {
55
-
throw new Exception(pht('This build log is not open!'));
56
-
}
57
-
58
-
$start = $this->getDateCreated();
59
-
$now = PhabricatorTime::getNow();
60
-
61
-
$this
62
-
->setDuration($now - $start)
63
-
->setLive(0)
64
-
->save();
65
-
66
-
$this->scheduleRebuild(false);
67
-
68
-
return $this;
45
+
->setLive(1);
69
46
}
70
47
71
48
public function scheduleRebuild($force) {
···
120
97
return pht('Build Log');
121
98
}
122
99
123
-
public function append($content) {
124
-
if (!$this->getLive()) {
125
-
throw new PhutilInvalidStateException('openBuildLog');
126
-
}
127
-
128
-
$content = (string)$content;
129
-
130
-
$this->rope->append($content);
131
-
$this->flush();
132
-
133
-
return $this;
134
-
}
135
-
136
-
private function flush() {
137
-
138
-
// TODO: Maybe don't flush more than a couple of times per second. If a
139
-
// caller writes a single character over and over again, we'll currently
140
-
// spend a lot of time flushing that.
141
-
142
-
$chunk_table = id(new HarbormasterBuildLogChunk())->getTableName();
143
-
$chunk_limit = self::CHUNK_BYTE_LIMIT;
144
-
$encoding_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
145
-
146
-
$rope = $this->rope;
147
-
148
-
while (true) {
149
-
$length = $rope->getByteLength();
150
-
if (!$length) {
151
-
break;
152
-
}
153
-
154
-
$conn_w = $this->establishConnection('w');
155
-
$last = $this->loadLastChunkInfo();
156
-
157
-
$can_append =
158
-
($last) &&
159
-
($last['encoding'] == $encoding_text) &&
160
-
($last['size'] < $chunk_limit);
161
-
if ($can_append) {
162
-
$append_id = $last['id'];
163
-
$prefix_size = $last['size'];
164
-
} else {
165
-
$append_id = null;
166
-
$prefix_size = 0;
167
-
}
168
-
169
-
$data_limit = ($chunk_limit - $prefix_size);
170
-
$append_data = $rope->getPrefixBytes($data_limit);
171
-
$data_size = strlen($append_data);
172
-
173
-
if ($append_id) {
174
-
queryfx(
175
-
$conn_w,
176
-
'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
177
-
$chunk_table,
178
-
$append_data,
179
-
$prefix_size + $data_size,
180
-
$append_id);
181
-
} else {
182
-
$this->writeChunk($encoding_text, $data_size, $append_data);
183
-
}
184
-
185
-
$rope->removeBytesFromHead($data_size);
186
-
}
187
-
}
188
-
189
100
public function newChunkIterator() {
190
101
return id(new HarbormasterBuildLogChunkIterator($this))
191
102
->setPageSize(8);
···
215
126
216
127
return implode('', $full_text);
217
128
}
129
+
130
+
131
+
public function getURI() {
132
+
$id = $this->getID();
133
+
return "/harbormaster/log/view/{$id}/";
134
+
}
135
+
136
+
137
+
/* -( Chunks )------------------------------------------------------------- */
138
+
218
139
219
140
public function canCompressLog() {
220
141
return function_exists('gzdeflate');
···
229
150
}
230
151
231
152
private function processLog($mode) {
153
+
if (!$this->getLock()->isLocked()) {
154
+
throw new Exception(
155
+
pht(
156
+
'You can not process build log chunks unless the log lock is '.
157
+
'held.'));
158
+
}
159
+
232
160
$chunks = $this->newChunkIterator();
233
161
234
162
// NOTE: Because we're going to insert new chunks, we need to stop the
···
292
220
->save();
293
221
}
294
222
295
-
public function getURI() {
296
-
$id = $this->getID();
297
-
return "/harbormaster/log/view/{$id}/";
223
+
224
+
/* -( Writing )------------------------------------------------------------ */
225
+
226
+
227
+
public function getLock() {
228
+
if (!$this->lock) {
229
+
$phid = $this->getPHID();
230
+
$phid_key = PhabricatorHash::digestToLength($phid, 14);
231
+
$lock_key = "build.log({$phid_key})";
232
+
$lock = PhabricatorGlobalLock::newLock($lock_key);
233
+
$this->lock = $lock;
234
+
}
235
+
236
+
return $this->lock;
237
+
}
238
+
239
+
240
+
public function openBuildLog() {
241
+
if ($this->isOpen) {
242
+
throw new Exception(pht('This build log is already open!'));
243
+
}
244
+
245
+
$is_new = !$this->getID();
246
+
if ($is_new) {
247
+
$this->save();
248
+
}
249
+
250
+
$this->getLock()->lock();
251
+
$this->isOpen = true;
252
+
253
+
$this->reload();
254
+
255
+
if (!$this->getLive()) {
256
+
$this->setLive(1)->save();
257
+
}
258
+
259
+
return $this;
260
+
}
261
+
262
+
public function closeBuildLog($forever = true) {
263
+
if (!$this->isOpen) {
264
+
throw new Exception(
265
+
pht(
266
+
'You must openBuildLog() before you can closeBuildLog().'));
267
+
}
268
+
269
+
$this->flush();
270
+
271
+
if ($forever) {
272
+
$start = $this->getDateCreated();
273
+
$now = PhabricatorTime::getNow();
274
+
275
+
$this
276
+
->setDuration($now - $start)
277
+
->setLive(0)
278
+
->save();
279
+
}
280
+
281
+
$this->getLock()->unlock();
282
+
$this->isOpen = false;
283
+
284
+
if ($forever) {
285
+
$this->scheduleRebuild(false);
286
+
}
287
+
288
+
return $this;
298
289
}
290
+
291
+
public function append($content) {
292
+
if (!$this->isOpen) {
293
+
throw new Exception(
294
+
pht(
295
+
'You must openBuildLog() before you can append() content to '.
296
+
'the log.'));
297
+
}
298
+
299
+
$content = (string)$content;
300
+
301
+
$this->rope->append($content);
302
+
$this->flush();
303
+
304
+
return $this;
305
+
}
306
+
307
+
private function flush() {
308
+
309
+
// TODO: Maybe don't flush more than a couple of times per second. If a
310
+
// caller writes a single character over and over again, we'll currently
311
+
// spend a lot of time flushing that.
312
+
313
+
$chunk_table = id(new HarbormasterBuildLogChunk())->getTableName();
314
+
$chunk_limit = self::CHUNK_BYTE_LIMIT;
315
+
$encoding_text = HarbormasterBuildLogChunk::CHUNK_ENCODING_TEXT;
316
+
317
+
$rope = $this->rope;
318
+
319
+
while (true) {
320
+
$length = $rope->getByteLength();
321
+
if (!$length) {
322
+
break;
323
+
}
324
+
325
+
$conn_w = $this->establishConnection('w');
326
+
$last = $this->loadLastChunkInfo();
327
+
328
+
$can_append =
329
+
($last) &&
330
+
($last['encoding'] == $encoding_text) &&
331
+
($last['size'] < $chunk_limit);
332
+
if ($can_append) {
333
+
$append_id = $last['id'];
334
+
$prefix_size = $last['size'];
335
+
} else {
336
+
$append_id = null;
337
+
$prefix_size = 0;
338
+
}
339
+
340
+
$data_limit = ($chunk_limit - $prefix_size);
341
+
$append_data = $rope->getPrefixBytes($data_limit);
342
+
$data_size = strlen($append_data);
343
+
344
+
if ($append_id) {
345
+
queryfx(
346
+
$conn_w,
347
+
'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d',
348
+
$chunk_table,
349
+
$append_data,
350
+
$prefix_size + $data_size,
351
+
$append_id);
352
+
} else {
353
+
$this->writeChunk($encoding_text, $data_size, $append_data);
354
+
}
355
+
356
+
$rope->removeBytesFromHead($data_size);
357
+
}
358
+
}
359
+
299
360
300
361
301
362
/* -( PhabricatorPolicyInterface )----------------------------------------- */
+13
-13
src/applications/harbormaster/worker/HarbormasterLogWorker.php
+13
-13
src/applications/harbormaster/worker/HarbormasterLogWorker.php
···
8
8
$data = $this->getTaskData();
9
9
$log_phid = idx($data, 'logPHID');
10
10
11
-
$phid_key = PhabricatorHash::digestToLength($log_phid, 14);
12
-
$lock_key = "build.log({$phid_key})";
13
-
$lock = PhabricatorGlobalLock::newLock($lock_key);
11
+
$log = id(new HarbormasterBuildLogQuery())
12
+
->setViewer($viewer)
13
+
->withPHIDs(array($log_phid))
14
+
->executeOne();
15
+
if (!$log) {
16
+
throw new PhabricatorWorkerPermanentFailureException(
17
+
pht(
18
+
'Invalid build log PHID "%s".',
19
+
$log_phid));
20
+
}
21
+
22
+
$lock = $log->getLock();
14
23
15
24
try {
16
25
$lock->lock();
···
20
29
21
30
$caught = null;
22
31
try {
23
-
$log = id(new HarbormasterBuildLogQuery())
24
-
->setViewer($viewer)
25
-
->withPHIDs(array($log_phid))
26
-
->executeOne();
27
-
if (!$log) {
28
-
throw new PhabricatorWorkerPermanentFailureException(
29
-
pht(
30
-
'Invalid build log PHID "%s".',
31
-
$log_phid));
32
-
}
32
+
$log->reload();
33
33
34
34
if ($log->getLive()) {
35
35
throw new PhabricatorWorkerPermanentFailureException(