@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a "metronome" for spreading service call load
Summary:
Ref T13244. See D20080. Rather than randomly jittering service calls, we can give each host a "metronome" that ticks every 60 seconds to get load to spread out after one cycle.
For example, web001 ticks (and makes a service call) when the second hand points at 0:17, web002 at 0:43, web003 at 0:04, etc.
For now I'm just planning to seed the metronomes randomly based on hostname, but we could conceivably give each host an assigned offset some day if we want perfectly smooth service call rates.
Test Plan: Ran unit tests.
Reviewers: amckinley
Reviewed By: amckinley
Maniphest Tasks: T13244
Differential Revision: https://secure.phabricator.com/D20087
+157
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
3552
3552
'PhabricatorMetaMTASchemaSpec' => 'applications/metamta/storage/PhabricatorMetaMTASchemaSpec.php',
3553
3553
'PhabricatorMetaMTASendGridReceiveController' => 'applications/metamta/controller/PhabricatorMetaMTASendGridReceiveController.php',
3554
3554
'PhabricatorMetaMTAWorker' => 'applications/metamta/PhabricatorMetaMTAWorker.php',
3555
+
'PhabricatorMetronome' => 'infrastructure/util/PhabricatorMetronome.php',
3556
+
'PhabricatorMetronomeTestCase' => 'infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php',
3555
3557
'PhabricatorMetronomicTriggerClock' => 'infrastructure/daemon/workers/clock/PhabricatorMetronomicTriggerClock.php',
3556
3558
'PhabricatorModularTransaction' => 'applications/transactions/storage/PhabricatorModularTransaction.php',
3557
3559
'PhabricatorModularTransactionType' => 'applications/transactions/storage/PhabricatorModularTransactionType.php',
···
9477
9479
'PhabricatorMetaMTASchemaSpec' => 'PhabricatorConfigSchemaSpec',
9478
9480
'PhabricatorMetaMTASendGridReceiveController' => 'PhabricatorMetaMTAController',
9479
9481
'PhabricatorMetaMTAWorker' => 'PhabricatorWorker',
9482
+
'PhabricatorMetronome' => 'Phobject',
9483
+
'PhabricatorMetronomeTestCase' => 'PhabricatorTestCase',
9480
9484
'PhabricatorMetronomicTriggerClock' => 'PhabricatorTriggerClock',
9481
9485
'PhabricatorModularTransaction' => 'PhabricatorApplicationTransaction',
9482
9486
'PhabricatorModularTransactionType' => 'Phobject',
+92
src/infrastructure/util/PhabricatorMetronome.php
+92
src/infrastructure/util/PhabricatorMetronome.php
···
1
+
<?php
2
+
3
+
/**
4
+
* Tick at a given frequency with a specifiable offset.
5
+
*
6
+
* One use case for this is to flatten out load spikes caused by periodic
7
+
* service calls. Give each host a metronome that ticks at the same frequency,
8
+
* but with different offsets. Then, have hosts make service calls only after
9
+
* their metronome ticks. This spreads service calls out evenly more quickly
10
+
* and more predictably than adding random jitter.
11
+
*/
12
+
final class PhabricatorMetronome
13
+
extends Phobject {
14
+
15
+
private $offset = 0;
16
+
private $frequency;
17
+
18
+
public function setOffset($offset) {
19
+
if (!is_int($offset)) {
20
+
throw new Exception(pht('Metronome offset must be an integer.'));
21
+
}
22
+
23
+
if ($offset < 0) {
24
+
throw new Exception(pht('Metronome offset must be 0 or more.'));
25
+
}
26
+
27
+
// We're not requiring that the offset be smaller than the frequency. If
28
+
// the offset is larger, we'll just clamp it to the frequency before we
29
+
// use it. This allows the offset to be configured before the frequency
30
+
// is configured, which is useful for using a hostname as an offset seed.
31
+
32
+
$this->offset = $offset;
33
+
34
+
return $this;
35
+
}
36
+
37
+
public function setFrequency($frequency) {
38
+
if (!is_int($frequency)) {
39
+
throw new Exception(pht('Metronome frequency must be an integer.'));
40
+
}
41
+
42
+
if ($frequency < 1) {
43
+
throw new Exception(pht('Metronome frequency must be 1 or more.'));
44
+
}
45
+
46
+
$this->frequency = $frequency;
47
+
48
+
return $this;
49
+
}
50
+
51
+
public function setOffsetFromSeed($seed) {
52
+
$offset = PhabricatorHash::digestToRange($seed, 0, PHP_INT_MAX);
53
+
return $this->setOffset($offset);
54
+
}
55
+
56
+
public function getFrequency() {
57
+
if ($this->frequency === null) {
58
+
throw new PhutilInvalidStateException('setFrequency');
59
+
}
60
+
return $this->frequency;
61
+
}
62
+
63
+
public function getOffset() {
64
+
$frequency = $this->getFrequency();
65
+
return ($this->offset % $frequency);
66
+
}
67
+
68
+
public function getNextTickAfter($epoch) {
69
+
$frequency = $this->getFrequency();
70
+
$offset = $this->getOffset();
71
+
72
+
$remainder = ($epoch % $frequency);
73
+
74
+
if ($remainder < $offset) {
75
+
return ($epoch - $remainder) + $offset;
76
+
} else {
77
+
return ($epoch - $remainder) + $frequency + $offset;
78
+
}
79
+
}
80
+
81
+
public function didTickBetween($min, $max) {
82
+
if ($max < $min) {
83
+
throw new Exception(
84
+
pht(
85
+
'Maximum tick window must not be smaller than minimum tick window.'));
86
+
}
87
+
88
+
$next = $this->getNextTickAfter($min);
89
+
return ($next <= $max);
90
+
}
91
+
92
+
}
+61
src/infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php
+61
src/infrastructure/util/__tests__/PhabricatorMetronomeTestCase.php
···
1
+
<?php
2
+
3
+
final class PhabricatorMetronomeTestCase
4
+
extends PhabricatorTestCase {
5
+
6
+
public function testMetronomeOffsets() {
7
+
$cases = array(
8
+
'web001.example.net' => 44,
9
+
'web002.example.net' => 36,
10
+
'web003.example.net' => 25,
11
+
'web004.example.net' => 25,
12
+
'web005.example.net' => 16,
13
+
'web006.example.net' => 26,
14
+
'web007.example.net' => 35,
15
+
'web008.example.net' => 14,
16
+
);
17
+
18
+
$metronome = id(new PhabricatorMetronome())
19
+
->setFrequency(60);
20
+
21
+
foreach ($cases as $input => $expect) {
22
+
$metronome->setOffsetFromSeed($input);
23
+
24
+
$this->assertEqual(
25
+
$expect,
26
+
$metronome->getOffset(),
27
+
pht('Offset for: %s', $input));
28
+
}
29
+
}
30
+
31
+
public function testMetronomeTicks() {
32
+
$metronome = id(new PhabricatorMetronome())
33
+
->setFrequency(60)
34
+
->setOffset(13);
35
+
36
+
$tick_epoch = strtotime('2000-01-01 11:11:13 AM UTC');
37
+
38
+
// Since the epoch is at "0:13" on the clock, the metronome should tick
39
+
// then.
40
+
$this->assertEqual(
41
+
$tick_epoch,
42
+
$metronome->getNextTickAfter($tick_epoch - 1),
43
+
pht('Tick at 11:11:13 AM.'));
44
+
45
+
// The next tick should be a minute later.
46
+
$this->assertEqual(
47
+
$tick_epoch + 60,
48
+
$metronome->getNextTickAfter($tick_epoch),
49
+
pht('Tick at 11:12:13 AM.'));
50
+
51
+
52
+
// There's no tick in the next 59 seconds.
53
+
$this->assertFalse(
54
+
$metronome->didTickBetween($tick_epoch, $tick_epoch + 59));
55
+
56
+
$this->assertTrue(
57
+
$metronome->didTickBetween($tick_epoch, $tick_epoch + 60));
58
+
}
59
+
60
+
61
+
}