getViewer(); $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withPHIDs(array($lease_phid)) ->executeOne(); if (!$lease) { throw new PhabricatorWorkerPermanentFailureException( pht('No such lease "%s"!', $lease_phid)); } return $lease; } protected function loadResource($resource_phid) { $viewer = $this->getViewer(); $resource = id(new DrydockResourceQuery()) ->setViewer($viewer) ->withPHIDs(array($resource_phid)) ->executeOne(); if (!$resource) { throw new PhabricatorWorkerPermanentFailureException( pht('No such resource "%s"!', $resource_phid)); } return $resource; } protected function loadOperation($operation_phid) { $viewer = $this->getViewer(); $operation = id(new DrydockRepositoryOperationQuery()) ->setViewer($viewer) ->withPHIDs(array($operation_phid)) ->executeOne(); if (!$operation) { throw new PhabricatorWorkerPermanentFailureException( pht('No such operation "%s"!', $operation_phid)); } return $operation; } protected function loadCommands($target_phid) { $viewer = $this->getViewer(); $commands = id(new DrydockCommandQuery()) ->setViewer($viewer) ->withTargetPHIDs(array($target_phid)) ->withConsumed(false) ->execute(); $commands = msort($commands, 'getID'); return $commands; } protected function checkLeaseExpiration(DrydockLease $lease) { $this->checkObjectExpiration($lease); } protected function checkResourceExpiration(DrydockResource $resource) { $this->checkObjectExpiration($resource); } private function checkObjectExpiration($object) { // Check if the resource or lease has expired. If it has, we're going to // send it a release command. // This command is sent from within the update worker so it is handled // immediately, but doing this generates a log and improves consistency. $expires = $object->getUntil(); if (!$expires) { return; } $now = PhabricatorTime::getNow(); if ($expires > $now) { return; } $viewer = $this->getViewer(); $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID(); $command = DrydockCommand::initializeNewCommand($viewer) ->setTargetPHID($object->getPHID()) ->setAuthorPHID($drydock_phid) ->setCommand(DrydockCommand::COMMAND_RELEASE) ->save(); } protected function yieldIfExpiringLease(DrydockLease $lease) { if (!$lease->canReceiveCommands()) { return; } $this->yieldIfExpiring($lease->getUntil()); } protected function yieldIfExpiringResource(DrydockResource $resource) { if (!$resource->canReceiveCommands()) { return; } $this->yieldIfExpiring($resource->getUntil()); } private function yieldIfExpiring($expires) { if (!$expires) { return; } if (!$this->getTaskDataValue('isExpireTask')) { return; } $now = PhabricatorTime::getNow(); throw new PhabricatorWorkerYieldException($expires - $now); } protected function isTemporaryException(Exception $ex) { if ($ex instanceof PhabricatorWorkerYieldException) { return true; } if ($ex instanceof DrydockSlotLockException) { return true; } if ($ex instanceof PhutilAggregateException) { $any_temporary = false; foreach ($ex->getExceptions() as $sub) { if ($this->isTemporaryException($sub)) { $any_temporary = true; break; } } if ($any_temporary) { return true; } } if ($ex->getPrevious()) { return $this->isTemporaryException($ex->getPrevious()); } return false; } protected function getYieldDurationFromException(Exception $ex) { if ($ex instanceof PhabricatorWorkerYieldException) { return $ex->getDuration(); } if ($ex instanceof DrydockSlotLockException) { return 5; } return 15; } protected function flushDrydockTaskQueue() { // NOTE: By default, queued tasks are not scheduled if the current task // fails. This is a good, safe default behavior. For example, it can // protect us from executing side effect tasks too many times, like // sending extra email. // However, it is not the behavior we want in Drydock, because we queue // followup tasks after lease and resource failures and want them to // execute in order to clean things up. // At least for now, we just explicitly flush the queue before exiting // with a failure to make sure tasks get queued up properly. try { $this->flushTaskQueue(); } catch (Exception $ex) { // If this fails, we want to swallow the exception so the caller throws // the original error, since we're more likely to be able to understand // and fix the problem if we have the original error than if we replace // it with this one. phlog($ex); } return $this; } protected function canReclaimResource(DrydockResource $resource) { $viewer = $this->getViewer(); // Don't reclaim a resource if it has been updated recently. If two // leases are fighting, we don't want them to keep reclaiming resources // from one another forever without making progress, so make resources // immune to reclamation for a little while after they activate or update. $now = PhabricatorTime::getNow(); $max_epoch = ($now - phutil_units('3 minutes in seconds')); // TODO: It would be nice to use a more narrow time here, like "last // activation or lease release", but we don't currently store that // anywhere. $updated = $resource->getDateModified(); if ($updated > $max_epoch) { return false; } $statuses = array( DrydockLeaseStatus::STATUS_PENDING, DrydockLeaseStatus::STATUS_ACQUIRED, DrydockLeaseStatus::STATUS_ACTIVE, DrydockLeaseStatus::STATUS_RELEASED, DrydockLeaseStatus::STATUS_BROKEN, ); // Don't reclaim resources that have any active leases. $leases = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withResourcePHIDs(array($resource->getPHID())) ->withStatuses($statuses) ->setLimit(1) ->execute(); if ($leases) { return false; } // See T13676. Don't reclaim a resource if a lease recently released. $leases = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withResourcePHIDs(array($resource->getPHID())) ->withStatuses( array( DrydockLeaseStatus::STATUS_DESTROYED, )) ->withDateModifiedBetween($max_epoch, null) ->setLimit(1) ->execute(); if ($leases) { return false; } return true; } protected function reclaimResource( DrydockResource $resource, DrydockLease $lease) { $viewer = $this->getViewer(); // Mark the lease as reclaiming this resource. It won't be allowed to start // another reclaim as long as this resource is still in the process of // being reclaimed. $lease->addReclaimedResourcePHIDs(array($resource->getPHID())); // When the resource releases, we we want to reawaken this task since it // should (usually) be able to start building a new resource right away. $worker_task_id = $this->getCurrentWorkerTaskID(); $command = DrydockCommand::initializeNewCommand($viewer) ->setTargetPHID($resource->getPHID()) ->setAuthorPHID($lease->getPHID()) ->setCommand(DrydockCommand::COMMAND_RECLAIM) ->setProperty('awakenTaskIDs', array($worker_task_id)); $lease->openTransaction(); $lease->save(); $command->save(); $lease->saveTransaction(); $resource->scheduleUpdate(); return $this; } }