getViewer(); if ($this->shouldLimitAllocatingPoolSize($blueprint)) { return false; } return true; } public function canAcquireLeaseOnResource( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // Don't hand out leases on working copies which have not activated, since // it may take an arbitrarily long time for them to acquire a host. if (!$resource->isActive()) { return false; } $need_map = $lease->getAttribute('repositories.map'); if (!is_array($need_map)) { return false; } $have_map = $resource->getAttribute('repositories.map'); if (!is_array($have_map)) { return false; } $have_as = ipull($have_map, 'phid'); $need_as = ipull($need_map, 'phid'); foreach ($need_as as $need_directory => $need_phid) { if (empty($have_as[$need_directory])) { // This resource is missing a required working copy. return false; } if ($have_as[$need_directory] != $need_phid) { // This resource has a required working copy, but it contains // the wrong repository. return false; } unset($have_as[$need_directory]); } if ($have_as && $lease->getAttribute('repositories.strict')) { // This resource has extra repositories, but the lease is strict about // which repositories are allowed to exist. return false; } if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) { return false; } return true; } public function acquireLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { $lease ->needSlotLock($this->getLeaseSlotLock($resource)) ->acquireOnResource($resource); } private function getLeaseSlotLock(DrydockResource $resource) { $resource_phid = $resource->getPHID(); return "workingcopy.lease({$resource_phid})"; } public function allocateResource( DrydockBlueprint $blueprint, DrydockLease $lease) { $resource = $this->newResourceTemplate($blueprint); $resource_phid = $resource->getPHID(); $blueprint_phids = $blueprint->getFieldValue('blueprintPHIDs'); $host_lease = $this->newLease($blueprint) ->setResourceType('host') ->setOwnerPHID($resource_phid) ->setAttribute('workingcopy.resourcePHID', $resource_phid) ->setAllowedBlueprintPHIDs($blueprint_phids); $resource->setAttribute('host.leasePHID', $host_lease->getPHID()); $map = $this->getWorkingCopyRepositoryMap($lease); $resource->setAttribute('repositories.map', $map); $slot_lock = $this->getConcurrentResourceLimitSlotLock($blueprint); if ($slot_lock !== null) { $resource->needSlotLock($slot_lock); } $resource->allocateResource(); $host_lease->queueForActivation(); return $resource; } private function getWorkingCopyRepositoryMap(DrydockLease $lease) { $attribute = 'repositories.map'; $map = $lease->getAttribute($attribute); // TODO: Leases should validate their attributes more formally. if (!is_array($map) || !$map) { $message = array(); if ($map === null) { $message[] = pht( 'Working copy lease is missing required attribute "%s".', $attribute); } else { $message[] = pht( 'Working copy lease has invalid attribute "%s".', $attribute); } $message[] = pht( 'Attribute "repositories.map" should be a map of repository '. 'specifications.'); $message = implode("\n\n", $message); throw new Exception($message); } foreach ($map as $key => $value) { $map[$key] = array_select_keys( $value, array( 'phid', )); } return $map; } public function activateResource( DrydockBlueprint $blueprint, DrydockResource $resource) { $lease = $this->loadHostLease($resource); $this->requireActiveLease($lease); $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $lease->getInterface($command_type); // TODO: Make this configurable. $resource_id = $resource->getID(); $root = "/var/drydock/workingcopy-{$resource_id}"; $map = $resource->getAttribute('repositories.map'); $futures = array(); $repositories = $this->loadRepositories(ipull($map, 'phid')); foreach ($map as $directory => $spec) { // TODO: Validate directory isn't goofy like "/etc" or "../../lol" // somewhere? $repository = $repositories[$spec['phid']]; $path = "{$root}/repo/{$directory}/"; $future = $interface->getExecFuture( 'git clone -- %s %s', (string)$repository->getCloneURIObject(), $path); $future->setTimeout($repository->getEffectiveCopyTimeLimit()); $futures[$directory] = $future; } foreach (new FutureIterator($futures) as $key => $future) { $future->resolvex(); } $resource ->setAttribute('workingcopy.root', $root) ->activateResource(); } public function destroyResource( DrydockBlueprint $blueprint, DrydockResource $resource) { try { $lease = $this->loadHostLease($resource); } catch (Exception $ex) { // If we can't load the lease, assume we don't need to take any actions // to destroy it. return; } // Destroy the lease on the host. $lease->setReleaseOnDestruction(true); if ($lease->isActive()) { // Destroy the working copy on disk. $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $lease->getInterface($command_type); $root_key = 'workingcopy.root'; $root = $resource->getAttribute($root_key); if (strlen($root)) { $interface->execx('rm -rf -- %s', $root); } } } public function getResourceName( DrydockBlueprint $blueprint, DrydockResource $resource) { return pht('Working Copy'); } public function activateLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { $host_lease = $this->loadHostLease($resource); $command_type = DrydockCommandInterface::INTERFACE_TYPE; $interface = $host_lease->getInterface($command_type); $map = $lease->getAttribute('repositories.map'); $root = $resource->getAttribute('workingcopy.root'); $repositories = $this->loadRepositories(ipull($map, 'phid')); $default = null; foreach ($map as $directory => $spec) { $repository = $repositories[$spec['phid']]; $interface->pushWorkingDirectory("{$root}/repo/{$directory}/"); $cmd = array(); $arg = array(); $cmd[] = 'git clean -d --force'; $cmd[] = 'git fetch'; $commit = idx($spec, 'commit'); $branch = idx($spec, 'branch'); $ref = idx($spec, 'ref'); // Reset things first, in case previous builds left anything staged or // dirty. Note that we don't reset to "HEAD" because that does not work // in empty repositories. $cmd[] = 'git reset --hard'; if ($commit !== null) { $cmd[] = 'git checkout %s --'; $arg[] = $commit; } else if ($branch !== null) { $cmd[] = 'git checkout %s --'; $arg[] = $branch; $cmd[] = 'git reset --hard origin/%s'; $arg[] = $branch; } $this->newExecvFuture($interface, $cmd, $arg) ->setTimeout($repository->getEffectiveCopyTimeLimit()) ->resolvex(); if (idx($spec, 'default')) { $default = $directory; } // If we're fetching a ref from a remote, do that separately so we can // raise a more tailored error. if ($ref) { $cmd = array(); $arg = array(); $ref_uri = $ref['uri']; $ref_ref = $ref['ref']; $cmd[] = 'git fetch --no-tags -- %s +%s:%s'; $arg[] = $ref_uri; $arg[] = $ref_ref; $arg[] = $ref_ref; $cmd[] = 'git checkout %s --'; $arg[] = $ref_ref; try { $this->newExecvFuture($interface, $cmd, $arg) ->setTimeout($repository->getEffectiveCopyTimeLimit()) ->resolvex(); } catch (CommandException $ex) { $display_command = csprintf( 'git fetch %R %R', $ref_uri, $ref_ref); $error = DrydockCommandError::newFromCommandException($ex) ->setPhase(self::PHASE_REMOTEFETCH) ->setDisplayCommand($display_command); $lease->setAttribute( 'workingcopy.vcs.error', $error->toDictionary()); throw $ex; } } $merges = idx($spec, 'merges'); if ($merges) { foreach ($merges as $merge) { $this->applyMerge($lease, $interface, $merge); } } $interface->popWorkingDirectory(); } if ($default === null) { $default = head_key($map); } // TODO: Use working storage? $lease->setAttribute('workingcopy.default', "{$root}/repo/{$default}/"); $lease->activateOnResource($resource); } public function didReleaseLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // We leave working copies around even if there are no leases on them, // since the cost to maintain them is nearly zero but rebuilding them is // moderately expensive and it's likely that they'll be reused. return; } public function destroyLease( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease) { // When we activate a lease we just reset the working copy state and do // not create any new state, so we don't need to do anything special when // destroying a lease. return; } public function getType() { return 'working-copy'; } public function getInterface( DrydockBlueprint $blueprint, DrydockResource $resource, DrydockLease $lease, $type) { switch ($type) { case DrydockCommandInterface::INTERFACE_TYPE: $host_lease = $this->loadHostLease($resource); $command_interface = $host_lease->getInterface($type); $path = $lease->getAttribute('workingcopy.default'); $command_interface->pushWorkingDirectory($path); return $command_interface; } } private function loadRepositories(array $phids) { $viewer = $this->getViewer(); $repositories = id(new PhabricatorRepositoryQuery()) ->setViewer($viewer) ->withPHIDs($phids) ->execute(); $repositories = mpull($repositories, null, 'getPHID'); foreach ($phids as $phid) { if (empty($repositories[$phid])) { throw new Exception( pht( 'Repository PHID "%s" does not exist.', $phid)); } } foreach ($repositories as $repository) { $repository_vcs = $repository->getVersionControlSystem(); switch ($repository_vcs) { case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: break; default: throw new Exception( pht( 'Repository ("%s") has unsupported VCS ("%s").', $repository->getPHID(), $repository_vcs)); } } return $repositories; } private function loadHostLease(DrydockResource $resource) { $viewer = $this->getViewer(); $lease_phid = $resource->getAttribute('host.leasePHID'); $lease = id(new DrydockLeaseQuery()) ->setViewer($viewer) ->withPHIDs(array($lease_phid)) ->executeOne(); if (!$lease) { throw new Exception( pht( 'Unable to load lease ("%s").', $lease_phid)); } return $lease; } protected function getCustomFieldSpecifications() { return array( 'blueprintPHIDs' => array( 'name' => pht('Use Blueprints'), 'type' => 'blueprints', 'required' => true, ), ); } protected function shouldUseConcurrentResourceLimit() { return true; } private function applyMerge( DrydockLease $lease, DrydockCommandInterface $interface, array $merge) { $src_uri = $merge['src.uri']; $src_ref = $merge['src.ref']; try { $interface->execx( 'git fetch --no-tags -- %s +%s:%s', $src_uri, $src_ref, $src_ref); } catch (CommandException $ex) { $display_command = csprintf( 'git fetch %R +%R:%R', $src_uri, $src_ref, $src_ref); $error = DrydockCommandError::newFromCommandException($ex) ->setPhase(self::PHASE_MERGEFETCH) ->setDisplayCommand($display_command); $lease->setAttribute('workingcopy.vcs.error', $error->toDictionary()); throw $ex; } // NOTE: This can never actually generate a commit because we pass // "--squash", but git sometimes runs code to check that a username and // email are configured anyway. $real_command = csprintf( 'git -c user.name=%s -c user.email=%s merge --no-stat --squash -- %R', 'drydock', 'drydock@phabricator', $src_ref); try { $interface->execx('%C', $real_command); } catch (CommandException $ex) { $display_command = csprintf( 'git merge --squash %R', $src_ref); $error = DrydockCommandError::newFromCommandException($ex) ->setPhase(self::PHASE_SQUASHMERGE) ->setDisplayCommand($display_command); $lease->setAttribute('workingcopy.vcs.error', $error->toDictionary()); throw $ex; } } public function getCommandError(DrydockLease $lease) { return $lease->getAttribute('workingcopy.vcs.error'); } private function execxv( DrydockCommandInterface $interface, array $commands, array $arguments) { return $this->newExecvFuture($interface, $commands, $arguments)->resolvex(); } private function newExecvFuture( DrydockCommandInterface $interface, array $commands, array $arguments) { $commands = implode(' && ', $commands); $argv = array_merge(array($commands), $arguments); return call_user_func_array(array($interface, 'getExecFuture'), $argv); } }