@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Show a queue utilization statistic in the Daemon console
Summary:
This came up recently in a discussion with @lifeihuang, and then tangentally with @hach-que. Make it easier for users to get a sense of whether they might need to add more daemons. Although we've improved the transparency of daemons, it's not easy for non-experts to determine at a glance how close to overflowing the queue is.
This number is approximate, but should be good enough for determining if your queue is more like 25% or 95% full.
If this goes over, say, 80%, it's probably a good idea to think about adding a couple of daemons. If it's under that, you should generally be fine.
Test Plan: {F88331}
Reviewers: btrahan, hach-que, lifeihuang
Reviewed By: btrahan
CC: hach-que, lifeihuang, aran, chad
Differential Revision: https://secure.phabricator.com/D7747
+55
-8
+54
-7
src/applications/daemon/controller/PhabricatorDaemonConsoleController.php
+54
-7
src/applications/daemon/controller/PhabricatorDaemonConsoleController.php
···
7
7
$request = $this->getRequest();
8
8
$user = $request->getUser();
9
9
10
+
$window_start = (time() - (60 * 15));
11
+
12
+
// Assume daemons spend about 250ms second in overhead per task acquiring
13
+
// leases and doing other bookkeeping. This is probably an over-estimation,
14
+
// but we'd rather show that utilization is too high than too low.
15
+
$lease_overhead = 0.250;
16
+
10
17
$completed = id(new PhabricatorWorkerArchiveTask())->loadAllWhere(
11
18
'dateModified > %d',
12
-
time() - (60 * 15));
19
+
$window_start);
13
20
14
21
$failed = id(new PhabricatorWorkerActiveTask())->loadAllWhere(
15
22
'failureTime > %d',
16
-
time() - (60 * 15));
23
+
$window_start);
24
+
25
+
$usage_total = 0;
26
+
$usage_start = PHP_INT_MAX;
17
27
18
28
$completed_info = array();
19
29
foreach ($completed as $completed_task) {
···
27
37
$completed_info[$class]['n']++;
28
38
$duration = $completed_task->getDuration();
29
39
$completed_info[$class]['duration'] += $duration;
40
+
41
+
// NOTE: Duration is in microseconds, but we're just using seconds to
42
+
// compute utilization.
43
+
$usage_total += $lease_overhead + ($duration / 1000000);
44
+
$usage_start = min($usage_start, $completed_task->getDateModified());
30
45
}
31
46
32
47
$completed_info = isort($completed_info, 'n');
···
41
56
}
42
57
43
58
if ($failed) {
59
+
// Add the time it takes to restart the daemons. This includes a guess
60
+
// about other overhead of 2X.
61
+
$usage_total += PhutilDaemonOverseer::RESTART_WAIT * count($failed) * 2;
62
+
foreach ($failed as $failed_task) {
63
+
$usage_start = min($usage_start, $failed_task->getFailureTime());
64
+
}
65
+
44
66
$rows[] = array(
45
67
phutil_tag('em', array(), pht('Temporary Failures')),
46
68
count($failed),
···
48
70
);
49
71
}
50
72
73
+
$logs = id(new PhabricatorDaemonLogQuery())
74
+
->setViewer($user)
75
+
->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE)
76
+
->execute();
77
+
78
+
$taskmasters = 0;
79
+
foreach ($logs as $log) {
80
+
if ($log->getDaemon() == 'PhabricatorTaskmasterDaemon') {
81
+
$taskmasters++;
82
+
}
83
+
}
84
+
85
+
if ($taskmasters && $usage_total) {
86
+
// Total number of wall-time seconds the daemons have been running since
87
+
// the oldest event. For very short times round up to 15s so we don't
88
+
// render any ridiculous numbers if you reload the page immediately after
89
+
// restarting the daemons.
90
+
$available_time = $taskmasters * max(15, (time() - $usage_start));
91
+
92
+
// Percentage of those wall-time seconds we can account for, which the
93
+
// daemons spent doing work:
94
+
$used_time = ($usage_total / $available_time);
95
+
96
+
$rows[] = array(
97
+
phutil_tag('em', array(), pht('Queue Utilization (Approximate)')),
98
+
sprintf('%.1f%%', 100 * $used_time),
99
+
null,
100
+
);
101
+
}
102
+
51
103
$completed_table = new AphrontTableView($rows);
52
104
$completed_table->setNoDataString(
53
105
pht('No tasks have completed in the last 15 minutes.'));
···
70
122
$completed_panel = new AphrontPanelView();
71
123
$completed_panel->appendChild($completed_table);
72
124
$completed_panel->setNoBackground();
73
-
74
-
$logs = id(new PhabricatorDaemonLogQuery())
75
-
->setViewer($user)
76
-
->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE)
77
-
->execute();
78
125
79
126
$daemon_header = id(new PHUIHeaderView())
80
127
->setHeader(pht('Active Daemons'));