@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Send graceful shutdown signals to daemons in Phabricator
Summary:
Fixes T5855. Adds a `--graceful N` flag to `phd stop` and `phd restart`.
`phd` will send SIGINT, wait `N` seconds, SIGTERM, wait 15 seconds, and SIGKILL. By default, `N` is 15.
Test Plan:
- Ran `bin/phd debug ...` and used `^C` to interrupt daemons. Saw graceful shutdown behavior, and abrupt termination on multiple `^C`.
- Ran `bin/phd start`, `bin/phd stop` and `bin/phd restart` with `--graceful` set to various things, notably `0`. Saw graceful shutdowns on the CLI and in the web UI. With `0`, abrupt shutdowns.
Reviewers: btrahan, hach-que
Reviewed By: hach-que
Subscribers: epriestley
Maniphest Tasks: T5855
Differential Revision: https://secure.phabricator.com/D10228
+117
-41
+8
src/applications/daemon/controller/PhabricatorDaemonLogViewController.php
+8
src/applications/daemon/controller/PhabricatorDaemonLogViewController.php
···
53
53
$tag->setBackgroundColor(PHUITagView::COLOR_BLUE);
54
54
$tag->setName(pht('Waiting'));
55
55
break;
56
+
case PhabricatorDaemonLog::STATUS_EXITING:
57
+
$tag->setBackgroundColor(PHUITagView::COLOR_YELLOW);
58
+
$tag->setName(pht('Exiting'));
59
+
break;
56
60
case PhabricatorDaemonLog::STATUS_EXITED:
57
61
$tag->setBackgroundColor(PHUITagView::COLOR_GREY);
58
62
$tag->setName(pht('Exited'));
···
135
139
'resuming work to avoid overloading services.',
136
140
phutil_format_relative_time($unknown_time),
137
141
phutil_format_relative_time($wait_time));
142
+
break;
143
+
case PhabricatorDaemonLog::STATUS_EXITING:
144
+
$details = pht(
145
+
'This daemon is shutting down gracefully.');
138
146
break;
139
147
case PhabricatorDaemonLog::STATUS_EXITED:
140
148
$details = pht(
+11
src/applications/daemon/event/PhabricatorDaemonEventListener.php
+11
src/applications/daemon/event/PhabricatorDaemonEventListener.php
···
8
8
$this->listen(PhutilDaemonOverseer::EVENT_DID_LAUNCH);
9
9
$this->listen(PhutilDaemonOverseer::EVENT_DID_LOG);
10
10
$this->listen(PhutilDaemonOverseer::EVENT_DID_HEARTBEAT);
11
+
$this->listen(PhutilDaemonOverseer::EVENT_WILL_GRACEFUL);
11
12
$this->listen(PhutilDaemonOverseer::EVENT_WILL_EXIT);
12
13
}
13
14
···
21
22
break;
22
23
case PhutilDaemonOverseer::EVENT_DID_LOG:
23
24
$this->handleLogEvent($event);
25
+
break;
26
+
case PhutilDaemonOverseer::EVENT_WILL_GRACEFUL:
27
+
$this->handleGracefulEvent($event);
24
28
break;
25
29
case PhutilDaemonOverseer::EVENT_WILL_EXIT:
26
30
$this->handleExitEvent($event);
···
84
88
if ($current_status !== $daemon->getStatus()) {
85
89
$daemon->setStatus($current_status)->save();
86
90
}
91
+
}
92
+
93
+
private function handleGracefulEvent(PhutilEvent $event) {
94
+
$id = $event->getValue('id');
95
+
96
+
$daemon = $this->getDaemon($id);
97
+
$daemon->setStatus(PhabricatorDaemonLog::STATUS_EXITING)->save();
87
98
}
88
99
89
100
private function handleExitEvent(PhutilEvent $event) {
+13
-2
src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php
+13
-2
src/applications/daemon/management/PhabricatorDaemonManagementRestartWorkflow.php
···
9
9
->setSynopsis(
10
10
pht(
11
11
'Stop, then start the standard daemon loadout.'))
12
-
->setArguments(array());
12
+
->setArguments(
13
+
array(
14
+
array(
15
+
'name' => 'graceful',
16
+
'param' => 'seconds',
17
+
'help' => pht(
18
+
'Grace period for daemons to attempt a clean shutdown, in '.
19
+
'seconds. Defaults to __15__ seconds.'),
20
+
'default' => 15,
21
+
),
22
+
));
13
23
}
14
24
15
25
public function execute(PhutilArgumentParser $args) {
16
-
$err = $this->executeStopCommand(array());
26
+
$graceful = $args->getArg('graceful');
27
+
$err = $this->executeStopCommand(array(), $graceful);
17
28
if ($err) {
18
29
return $err;
19
30
}
+10
-1
src/applications/daemon/management/PhabricatorDaemonManagementStopWorkflow.php
+10
-1
src/applications/daemon/management/PhabricatorDaemonManagementStopWorkflow.php
···
13
13
->setArguments(
14
14
array(
15
15
array(
16
+
'name' => 'graceful',
17
+
'param' => 'seconds',
18
+
'help' => pht(
19
+
'Grace period for daemons to attempt a clean shutdown, in '.
20
+
'seconds. Defaults to __15__ seconds.'),
21
+
'default' => 15,
22
+
),
23
+
array(
16
24
'name' => 'pids',
17
25
'wildcard' => true,
18
26
),
···
21
29
22
30
public function execute(PhutilArgumentParser $args) {
23
31
$pids = $args->getArg('pids');
24
-
return $this->executeStopCommand($pids);
32
+
$graceful = $args->getArg('graceful');
33
+
return $this->executeStopCommand($pids, $graceful);
25
34
}
26
35
27
36
}
+54
-30
src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php
+54
-30
src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php
···
286
286
return 0;
287
287
}
288
288
289
-
protected final function executeStopCommand(array $pids) {
289
+
protected final function executeStopCommand(array $pids, $grace_period) {
290
290
$console = PhutilConsole::getConsole();
291
291
292
292
$daemons = $this->loadRunningDaemons();
···
325
325
}
326
326
327
327
$all_daemons = $running;
328
-
foreach ($running as $key => $daemon) {
329
-
$pid = $daemon->getPID();
330
-
$name = $daemon->getName();
331
328
332
-
$console->writeErr(pht("Stopping daemon '%s' (%s)...", $name, $pid)."\n");
333
-
if (!$daemon->isRunning()) {
334
-
$console->writeErr(pht('Daemon is not running.')."\n");
335
-
unset($running[$key]);
336
-
$daemon->updateStatus(PhabricatorDaemonLog::STATUS_EXITED);
337
-
} else {
338
-
posix_kill($pid, SIGINT);
339
-
}
329
+
// If we're doing a graceful shutdown, try SIGINT first.
330
+
if ($grace_period) {
331
+
$running = $this->sendSignal($running, SIGINT, $grace_period);
340
332
}
341
333
342
-
$start = time();
343
-
do {
344
-
foreach ($running as $key => $daemon) {
345
-
$pid = $daemon->getPID();
346
-
if (!$daemon->isRunning()) {
347
-
$console->writeOut(pht('Daemon %s exited normally.', $pid)."\n");
348
-
unset($running[$key]);
349
-
}
350
-
}
351
-
if (empty($running)) {
352
-
break;
353
-
}
354
-
usleep(100000);
355
-
} while (time() < $start + 15);
334
+
// If we still have daemons, SIGTERM them.
335
+
if ($running) {
336
+
$running = $this->sendSignal($running, SIGTERM, 15);
337
+
}
356
338
357
-
foreach ($running as $key => $daemon) {
358
-
$pid = $daemon->getPID();
359
-
$console->writeErr(pht('Sending daemon %s a SIGKILL.', $pid)."\n");
360
-
posix_kill($pid, SIGKILL);
339
+
// If the overseer is still alive, SIGKILL it.
340
+
if ($running) {
341
+
$this->sendSignal($running, SIGKILL, 0);
361
342
}
362
343
363
344
foreach ($all_daemons as $daemon) {
···
367
348
}
368
349
369
350
return 0;
351
+
}
352
+
353
+
private function sendSignal(array $daemons, $signo, $wait) {
354
+
$console = PhutilConsole::getConsole();
355
+
356
+
foreach ($daemons as $key => $daemon) {
357
+
$pid = $daemon->getPID();
358
+
$name = $daemon->getName();
359
+
360
+
switch ($signo) {
361
+
case SIGINT:
362
+
$message = pht("Interrupting daemon '%s' (%s)...", $name, $pid);
363
+
break;
364
+
case SIGTERM:
365
+
$message = pht("Terminating daemon '%s' (%s)...", $name, $pid);
366
+
break;
367
+
case SIGKILL:
368
+
$message = pht("Killing daemon '%s' (%s)...", $name, $pid);
369
+
break;
370
+
}
371
+
372
+
$console->writeOut("%s\n", $message);
373
+
posix_kill($pid, $signo);
374
+
}
375
+
376
+
if ($wait) {
377
+
$start = PhabricatorTime::getNow();
378
+
do {
379
+
foreach ($daemons as $key => $daemon) {
380
+
$pid = $daemon->getPID();
381
+
if (!$daemon->isRunning()) {
382
+
$console->writeOut(pht('Daemon %s exited.', $pid)."\n");
383
+
unset($daemons[$key]);
384
+
}
385
+
}
386
+
if (empty($daemons)) {
387
+
break;
388
+
}
389
+
usleep(100000);
390
+
} while (PhabricatorTime::getNow() < $start + $wait);
391
+
}
392
+
393
+
return $daemons;
370
394
}
371
395
372
396
private function freeActiveLeases() {
+4
-1
src/applications/daemon/query/PhabricatorDaemonLogQuery.php
+4
-1
src/applications/daemon/query/PhabricatorDaemonLogQuery.php
···
67
67
$status_running = PhabricatorDaemonLog::STATUS_RUNNING;
68
68
$status_unknown = PhabricatorDaemonLog::STATUS_UNKNOWN;
69
69
$status_wait = PhabricatorDaemonLog::STATUS_WAIT;
70
+
$status_exiting = PhabricatorDaemonLog::STATUS_EXITING;
70
71
$status_exited = PhabricatorDaemonLog::STATUS_EXITED;
71
72
$status_dead = PhabricatorDaemonLog::STATUS_DEAD;
72
73
···
77
78
$seen = $daemon->getDateModified();
78
79
79
80
$is_running = ($status == $status_running) ||
80
-
($status == $status_wait);
81
+
($status == $status_wait) ||
82
+
($status == $status_exiting);
81
83
82
84
// If we haven't seen the daemon recently, downgrade its status to
83
85
// unknown.
···
160
162
PhabricatorDaemonLog::STATUS_UNKNOWN,
161
163
PhabricatorDaemonLog::STATUS_RUNNING,
162
164
PhabricatorDaemonLog::STATUS_WAIT,
165
+
PhabricatorDaemonLog::STATUS_EXITING,
163
166
);
164
167
default:
165
168
throw new Exception("Unknown status '{$status}'!");
+1
src/applications/daemon/storage/PhabricatorDaemonLog.php
+1
src/applications/daemon/storage/PhabricatorDaemonLog.php
+5
-2
src/applications/daemon/view/PhabricatorDaemonLogListView.php
+5
-2
src/applications/daemon/view/PhabricatorDaemonLogListView.php
···
17
17
throw new Exception('Call setUser() before rendering!');
18
18
}
19
19
20
-
$list = id(new PHUIObjectItemListView())
21
-
->setFlush(true);
20
+
$list = new PHUIObjectItemListView();
22
21
foreach ($this->daemonLogs as $log) {
23
22
$id = $log->getID();
24
23
$epoch = $log->getDateCreated();
···
42
41
'This daemon is lost or exited uncleanly, and is presumed '.
43
42
'dead.'));
44
43
$item->addIcon('fa-times grey', pht('Dead'));
44
+
break;
45
+
case PhabricatorDaemonLog::STATUS_EXITING:
46
+
$item->addAttribute(pht('This daemon is exiting.'));
47
+
$item->addIcon('fa-check', pht('Exiting'));
45
48
break;
46
49
case PhabricatorDaemonLog::STATUS_EXITED:
47
50
$item->setDisabled(true);
+1
-1
src/applications/fact/daemon/PhabricatorFactDaemon.php
+1
-1
src/applications/fact/daemon/PhabricatorFactDaemon.php
···
8
8
9
9
public function run() {
10
10
$this->setEngines(PhabricatorFactEngine::loadAllEngines());
11
-
while (true) {
11
+
while (!$this->shouldExit()) {
12
12
$iterators = $this->getAllApplicationIterators();
13
13
foreach ($iterators as $iterator_name => $iterator) {
14
14
$this->processIteratorWithCursor($iterator_name, $iterator);
+7
-1
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
+7
-1
src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
···
71
71
$futures = array();
72
72
$queue = array();
73
73
74
-
while (true) {
74
+
while (!$this->shouldExit()) {
75
75
$pullable = $this->loadPullableRepositories($include, $exclude);
76
76
77
77
// If any repositories have the NEEDS_UPDATE flag set, pull them
···
422
422
new PhutilNumber($sleep_duration)));
423
423
424
424
$this->sleep(1);
425
+
426
+
if ($this->shouldExit()) {
427
+
$this->log(pht('Awakened from sleep by graceful shutdown!'));
428
+
return;
429
+
}
430
+
425
431
if ($this->loadRepositoryUpdateMessages()) {
426
432
$this->log(pht('Awakened from sleep by pending updates!'));
427
433
break;
+1
-1
src/infrastructure/daemon/bot/PhabricatorBot.php
+1
-1
src/infrastructure/daemon/bot/PhabricatorBot.php
+1
-1
src/infrastructure/daemon/garbagecollector/PhabricatorGarbageCollectorDaemon.php
+1
-1
src/infrastructure/daemon/garbagecollector/PhabricatorGarbageCollectorDaemon.php