Make `phd` more aware of multiple daemons under a single overseer

+95 -129

2 changed files

expand all

src

applications

daemon

management

PhabricatorDaemonManagementWorkflow.php

infrastructure

daemon

control

PhabricatorDaemonReference.php

+63 -58

src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php

··· 44 44 $pid_files = Filesystem::listDirectory($pid_dir); 45 45 46 46 foreach ($pid_files as $pid_file) { 47 - $daemons[] = PhabricatorDaemonReference::newFromFile( 48 - $pid_dir.'/'.$pid_file); 47 + $path = $pid_dir.'/'.$pid_file; 48 + $daemons[] = PhabricatorDaemonReference::loadReferencesFromFile($path); 49 49 } 50 50 51 - return $daemons; 51 + return array_mergev($daemons); 52 52 } 53 53 54 54 protected final function loadAllRunningDaemons() { ··· 403 403 return 0; 404 404 } 405 405 406 - $daemons = mpull($daemons, null, 'getPID'); 407 - 408 - $running = array(); 406 + $running_pids = array_fuse(mpull($daemons, 'getPID')); 409 407 if (!$pids) { 410 - $running = $daemons; 408 + $stop_pids = $running_pids; 411 409 } else { 412 410 // We were given a PID or set of PIDs to kill. 411 + $stop_pids = array(); 413 412 foreach ($pids as $key => $pid) { 414 413 if (!preg_match('/^\d+$/', $pid)) { 415 414 $console->writeErr(pht("PID '%s' is not a valid PID.", $pid)."\n"); 416 415 continue; 417 - } else if (empty($daemons[$pid])) { 416 + } else if (empty($running_pids[$pid])) { 418 417 $console->writeErr( 419 418 pht( 420 - "PID '%s' is not a Phabricator daemon PID. It will not ". 421 - "be killed.", 419 + 'PID "%d" is not a known Phabricator daemon PID. It will not '. 420 + 'be killed.', 422 421 $pid)."\n"); 423 422 continue; 424 423 } else { 425 - $running[] = $daemons[$pid]; 424 + $stop_pids[$pid] = $pid; 426 425 } 427 426 } 428 427 } 429 428 430 - if (empty($running)) { 429 + if (!$stop_pids) { 431 430 $console->writeErr(pht('No daemons to kill.')."\n"); 432 431 return 0; 433 432 } 434 433 435 - $all_daemons = $running; 436 - // don't specify force here as that's about rogue daemons 437 - $this->sendStopSignals($running, $grace_period); 434 + $survivors = $this->sendStopSignals($stop_pids, $grace_period); 438 435 439 - foreach ($all_daemons as $daemon) { 440 - if ($daemon->getPIDFile()) { 441 - Filesystem::remove($daemon->getPIDFile()); 436 + // Try to clean up PID files for daemons we killed. 437 + $remove = array(); 438 + foreach ($daemons as $daemon) { 439 + $pid = $daemon->getPID(); 440 + if (empty($stop_pids[$pid])) { 441 + // We did not try to stop this overseer. 442 + continue; 442 443 } 444 + 445 + if (isset($survivors[$pid])) { 446 + // We weren't able to stop this overseer. 447 + continue; 448 + } 449 + 450 + if (!$daemon->getPIDFile()) { 451 + // We don't know where the PID file is. 452 + continue; 453 + } 454 + 455 + $remove[] = $daemon->getPIDFile(); 456 + } 457 + 458 + foreach (array_unique($remove) as $remove_file) { 459 + Filesystem::remove($remove_file); 443 460 } 444 461 445 462 if (!$gently) { ··· 455 472 $rogue_daemons = PhutilDaemonOverseer::findRunningDaemons(); 456 473 if ($rogue_daemons) { 457 474 if ($force_stop) { 458 - $stop_rogue_daemons = $this->buildRogueDaemons($rogue_daemons); 459 - $survivors = $this->sendStopSignals( 460 - $stop_rogue_daemons, 461 - $grace_period, 462 - $force_stop); 475 + $rogue_pids = ipull($rogue_daemons, 'pid'); 476 + $survivors = $this->sendStopSignals($rogue_pids, $grace_period); 463 477 if ($survivors) { 464 - $console->writeErr(pht( 465 - 'Unable to stop processes running without pid files. Try running '. 466 - 'this command again with sudo.'."\n")); 478 + $console->writeErr( 479 + pht( 480 + 'Unable to stop processes running without PID files. '. 481 + 'Try running this command again with sudo.')."\n"); 467 482 } 468 483 } else if ($warn) { 469 484 $console->writeErr($this->getForceStopHint($rogue_daemons)."\n"); 470 485 } 471 486 } 487 + 472 488 return $rogue_daemons; 473 489 } 474 490 ··· 485 501 $debug_output); 486 502 } 487 503 488 - private function buildRogueDaemons(array $daemons) { 489 - $rogue_daemons = array(); 490 - foreach ($daemons as $pid => $data) { 491 - $rogue_daemons[] = 492 - PhabricatorDaemonReference::newFromRogueDictionary($data); 493 - } 494 - return $rogue_daemons; 495 - } 496 - 497 - private function sendStopSignals($daemons, $grace_period, $force = false) { 504 + private function sendStopSignals($pids, $grace_period) { 498 505 // If we're doing a graceful shutdown, try SIGINT first. 499 506 if ($grace_period) { 500 - $daemons = $this->sendSignal($daemons, SIGINT, $grace_period, $force); 507 + $pids = $this->sendSignal($pids, SIGINT, $grace_period); 501 508 } 502 509 503 510 // If we still have daemons, SIGTERM them. 504 - if ($daemons) { 505 - $daemons = $this->sendSignal($daemons, SIGTERM, 15, $force); 511 + if ($pids) { 512 + $pids = $this->sendSignal($pids, SIGTERM, 15); 506 513 } 507 514 508 515 // If the overseer is still alive, SIGKILL it. 509 - if ($daemons) { 510 - $daemons = $this->sendSignal($daemons, SIGKILL, 0, $force); 516 + if ($pids) { 517 + $pids = $this->sendSignal($pids, SIGKILL, 0); 511 518 } 512 - return $daemons; 519 + 520 + return $pids; 513 521 } 514 522 515 - private function sendSignal(array $daemons, $signo, $wait, $force = false) { 523 + private function sendSignal(array $pids, $signo, $wait) { 516 524 $console = PhutilConsole::getConsole(); 517 525 518 - foreach ($daemons as $key => $daemon) { 519 - $pid = $daemon->getPID(); 520 - $name = $daemon->getName(); 526 + $pids = array_fuse($pids); 521 527 528 + foreach ($pids as $key => $pid) { 522 529 if (!$pid) { 523 530 // NOTE: We must have a PID to signal a daemon, since sending a signal 524 531 // to PID 0 kills this process. 525 - $console->writeOut("%s\n", pht("Daemon '%s' has no PID!", $name)); 526 - unset($daemons[$key]); 532 + unset($pids[$key]); 527 533 continue; 528 534 } 529 535 530 536 switch ($signo) { 531 537 case SIGINT: 532 - $message = pht("Interrupting daemon '%s' (%s)...", $name, $pid); 538 + $message = pht('Interrupting process %d...', $pid); 533 539 break; 534 540 case SIGTERM: 535 - $message = pht("Terminating daemon '%s' (%s)...", $name, $pid); 541 + $message = pht('Terminating process %d...', $pid); 536 542 break; 537 543 case SIGKILL: 538 - $message = pht("Killing daemon '%s' (%s)...", $name, $pid); 544 + $message = pht('Killing process %d...', $pid); 539 545 break; 540 546 } 541 547 ··· 546 552 if ($wait) { 547 553 $start = PhabricatorTime::getNow(); 548 554 do { 549 - foreach ($daemons as $key => $daemon) { 550 - $pid = $daemon->getPID(); 551 - if (!$daemon->isRunning()) { 552 - $console->writeOut(pht('Daemon %s exited.', $pid)."\n"); 553 - unset($daemons[$key]); 555 + foreach ($pids as $key => $pid) { 556 + if (!PhabricatorDaemonReference::isProcessRunning($pid)) { 557 + $console->writeOut(pht('Process %d exited.', $pid)."\n"); 558 + unset($pids[$key]); 554 559 } 555 560 } 556 - if (empty($daemons)) { 561 + if (empty($pids)) { 557 562 break; 558 563 } 559 564 usleep(100000); 560 565 } while (PhabricatorTime::getNow() < $start + $wait); 561 566 } 562 567 563 - return $daemons; 568 + return $pids; 564 569 } 565 570 566 571 private function freeActiveLeases() {

+32 -71

src/infrastructure/daemon/control/PhabricatorDaemonReference.php

··· 10 10 11 11 private $daemonLog; 12 12 13 - public static function newFromFile($path) { 13 + public static function loadReferencesFromFile($path) { 14 14 $pid_data = Filesystem::readFile($path); 15 15 16 16 try { ··· 19 19 $dict = array(); 20 20 } 21 21 22 - $ref = self::newFromDictionary($dict); 23 - $ref->pidFile = $path; 24 - return $ref; 25 - } 22 + $refs = array(); 23 + $daemons = idx($dict, 'daemons', array()); 26 24 27 - public static function newFromDictionary(array $dict) { 28 - $ref = new PhabricatorDaemonReference(); 25 + foreach ($daemons as $daemon) { 26 + $ref = new PhabricatorDaemonReference(); 29 27 30 - // TODO: This is a little rough during the transition from one-to-one 31 - // overseers to one-to-many. 32 - $config = idx($dict, 'config', array()); 28 + // NOTE: This is the overseer PID, not the actual daemon process PID. 29 + // This is correct for checking status and sending signals (the only 30 + // things we do with it), but might be confusing. $daemon['pid'] has 31 + // the daemon PID, and we could expose that if we had some use for it. 33 32 34 - $daemon_list = null; 35 - if ($config) { 36 - $daemon_list = idx($config, 'daemons'); 37 - } 33 + $ref->pid = idx($dict, 'pid'); 34 + $ref->start = idx($dict, 'start'); 38 35 39 - if ($daemon_list) { 40 - $ref->name = pht('Overseer Daemon Group'); 41 - $ref->argv = array(); 42 - } else { 43 - $ref->name = idx($dict, 'name', 'Unknown'); 44 - $ref->argv = idx($dict, 'argv', array()); 45 - } 36 + $ref->name = idx($daemon, 'class'); 37 + $ref->argv = idx($daemon, 'argv', array()); 46 38 47 - $ref->pid = idx($dict, 'pid'); 48 - $ref->start = idx($dict, 'start'); 49 39 50 - try { 51 - $ref->daemonLog = id(new PhabricatorDaemonLog())->loadOneWhere( 52 - 'daemon = %s AND pid = %d AND dateCreated = %d', 53 - $ref->name, 54 - $ref->pid, 55 - $ref->start); 56 - } catch (AphrontQueryException $ex) { 57 - // Ignore the exception. We want to be able to terminate the daemons, 58 - // even if MySQL is down. 40 + // TODO: We previously identified daemon logs by using a <class, pid, 41 + // epoch> tuple, but now all daemons under a single overseer will share 42 + // that identifier. We can uniquely identify daemons by $daemon['id'], 43 + // but that isn't currently written into the daemon logs. We should 44 + // start writing it, then load the logs here. This would give us a 45 + // slightly greater ability to keep the web UI in sync when daemons 46 + // get killed forcefully and clean up `phd status` a bit. 47 + 48 + $ref->pidFile = $path; 49 + $refs[] = $ref; 59 50 } 60 51 61 - return $ref; 62 - } 63 - 64 - /** 65 - * Appropriate for getting @{class:PhabricatorDaemonReference} objects from 66 - * the data from @{class:PhabricatorDaemonManagementWorkflow}'s method 67 - * @{method:findRunningDaemons}. 68 - * 69 - * NOTE: the objects are not fully featured and should be used with caution. 70 - */ 71 - public static function newFromRogueDictionary(array $dict) { 72 - $ref = new PhabricatorDaemonReference(); 73 - $ref->name = pht('Rogue %s', idx($dict, 'type')); 74 - $ref->pid = idx($dict, 'pid'); 75 - 76 - return $ref; 52 + return $refs; 77 53 } 78 54 79 55 public function updateStatus($new_status) { 56 + if (!$this->daemonLog) { 57 + return; 58 + } 59 + 80 60 try { 81 - if (!$this->daemonLog) { 82 - $this->daemonLog = id(new PhabricatorDaemonLog())->loadOneWhere( 83 - 'daemon = %s AND pid = %d AND dateCreated = %d', 84 - $this->name, 85 - $this->pid, 86 - $this->start); 87 - } 88 - 89 - if ($this->daemonLog) { 90 - $this->daemonLog 91 - ->setStatus($new_status) 92 - ->save(); 93 - } 61 + $this->daemonLog 62 + ->setStatus($new_status) 63 + ->save(); 94 64 } catch (AphrontQueryException $ex) { 95 - // Ignore anything that goes wrong here. We anticipate at least two 96 - // specific failure modes: 97 - // 98 - // - Upgrade scripts which run `git pull`, then `phd stop`, then 99 - // `bin/storage upgrade` will fail when trying to update the `status` 100 - // column, as it does not exist yet. 101 - // - Daemons running on machines which do not have access to MySQL 102 - // (like an IRC bot) will not be able to load or save the log. 103 - // 104 - // 65 + // Ignore anything that goes wrong here. 105 66 } 106 67 } 107 68

Configure Feed

Configure Feed