recaptime.dev / phorge
@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.) hq.recaptime.dev/wiki/Phorge
phorge phabricator
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Support working copies and separate allocate + activate steps for resources/leases in Drydock

Summary:
Ref T9253. For resources and leases that need to do something which takes a lot of time or requires waiting, allow them to allocate/acquire first and then activate later.

When we allocate a resource or acquire a lease, the blueprint can either activate it immediately (if all the work can happen quickly/inline) or activate it later. If the blueprint activates it later, we queue a worker to handle activating it.

Rebuild the "working copy" blueprint to work with this model: it allocates/acquires and activates in a separate step, once it is able to acquire a host.

Test Plan: With some power of imagination, brought up a bunch of working copies with `bin/drydock lease --type working-copy ...`

Reviewers: hach-que, chad

Reviewed By: hach-que, chad

Maniphest Tasks: T9253

Differential Revision: https://secure.phabricator.com/D14127

epriestley f1119ffc 6a0eb9d8

+628 -128
+9 -1
src/__phutil_library_map__.php
··· 838 838 'DrydockLeaseSearchEngine' => 'applications/drydock/query/DrydockLeaseSearchEngine.php', 839 839 'DrydockLeaseStatus' => 'applications/drydock/constants/DrydockLeaseStatus.php', 840 840 'DrydockLeaseViewController' => 'applications/drydock/controller/DrydockLeaseViewController.php', 841 + 'DrydockLeaseWorker' => 'applications/drydock/worker/DrydockLeaseWorker.php', 841 842 'DrydockLog' => 'applications/drydock/storage/DrydockLog.php', 842 843 'DrydockLogController' => 'applications/drydock/controller/DrydockLogController.php', 843 844 'DrydockLogListController' => 'applications/drydock/controller/DrydockLogListController.php', ··· 861 862 'DrydockResourceSearchEngine' => 'applications/drydock/query/DrydockResourceSearchEngine.php', 862 863 'DrydockResourceStatus' => 'applications/drydock/constants/DrydockResourceStatus.php', 863 864 'DrydockResourceViewController' => 'applications/drydock/controller/DrydockResourceViewController.php', 865 + 'DrydockResourceWorker' => 'applications/drydock/worker/DrydockResourceWorker.php', 864 866 'DrydockSFTPFilesystemInterface' => 'applications/drydock/interface/filesystem/DrydockSFTPFilesystemInterface.php', 865 867 'DrydockSSHCommandInterface' => 'applications/drydock/interface/command/DrydockSSHCommandInterface.php', 866 868 'DrydockSlotLock' => 'applications/drydock/storage/DrydockSlotLock.php', 869 + 'DrydockSlotLockException' => 'applications/drydock/exception/DrydockSlotLockException.php', 867 870 'DrydockWebrootInterface' => 'applications/drydock/interface/webroot/DrydockWebrootInterface.php', 871 + 'DrydockWorker' => 'applications/drydock/worker/DrydockWorker.php', 868 872 'DrydockWorkingCopyBlueprintImplementation' => 'applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php', 869 873 'FeedConduitAPIMethod' => 'applications/feed/conduit/FeedConduitAPIMethod.php', 870 874 'FeedPublishConduitAPIMethod' => 'applications/feed/conduit/FeedPublishConduitAPIMethod.php', ··· 4502 4506 'DoorkeeperSchemaSpec' => 'PhabricatorConfigSchemaSpec', 4503 4507 'DoorkeeperTagView' => 'AphrontView', 4504 4508 'DoorkeeperTagsController' => 'PhabricatorController', 4505 - 'DrydockAllocatorWorker' => 'PhabricatorWorker', 4509 + 'DrydockAllocatorWorker' => 'DrydockWorker', 4506 4510 'DrydockAlmanacServiceHostBlueprintImplementation' => 'DrydockBlueprintImplementation', 4507 4511 'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface', 4508 4512 'DrydockBlueprint' => array( ··· 4555 4559 'DrydockLeaseSearchEngine' => 'PhabricatorApplicationSearchEngine', 4556 4560 'DrydockLeaseStatus' => 'DrydockConstants', 4557 4561 'DrydockLeaseViewController' => 'DrydockLeaseController', 4562 + 'DrydockLeaseWorker' => 'DrydockWorker', 4558 4563 'DrydockLog' => array( 4559 4564 'DrydockDAO', 4560 4565 'PhabricatorPolicyInterface', ··· 4584 4589 'DrydockResourceSearchEngine' => 'PhabricatorApplicationSearchEngine', 4585 4590 'DrydockResourceStatus' => 'DrydockConstants', 4586 4591 'DrydockResourceViewController' => 'DrydockResourceController', 4592 + 'DrydockResourceWorker' => 'DrydockWorker', 4587 4593 'DrydockSFTPFilesystemInterface' => 'DrydockFilesystemInterface', 4588 4594 'DrydockSSHCommandInterface' => 'DrydockCommandInterface', 4589 4595 'DrydockSlotLock' => 'DrydockDAO', 4596 + 'DrydockSlotLockException' => 'Exception', 4590 4597 'DrydockWebrootInterface' => 'DrydockInterface', 4598 + 'DrydockWorker' => 'PhabricatorWorker', 4591 4599 'DrydockWorkingCopyBlueprintImplementation' => 'DrydockBlueprintImplementation', 4592 4600 'FeedConduitAPIMethod' => 'ConduitAPIMethod', 4593 4601 'FeedPublishConduitAPIMethod' => 'FeedConduitAPIMethod',
+10 -9
src/applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php
··· 76 76 ->needSlotLock("almanac.host.binding({$binding_phid})"); 77 77 78 78 try { 79 - return $resource->allocateResource(DrydockResourceStatus::STATUS_OPEN); 79 + return $resource->allocateResource(); 80 80 } catch (Exception $ex) { 81 81 $exceptions[] = $ex; 82 82 } ··· 92 92 DrydockResource $resource, 93 93 DrydockLease $lease) { 94 94 95 - // TODO: The current rule is one lease per resource, and there's no way to 96 - // make that cheaper here than by just trying to acquire the lease below, 97 - // so don't do any special checks for now. When we eventually permit 98 - // multiple leases per host, we'll need to load leases anyway, so we can 99 - // reject fully leased hosts cheaply here. 95 + if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) { 96 + return false; 97 + } 100 98 101 99 return true; 102 100 } ··· 106 104 DrydockResource $resource, 107 105 DrydockLease $lease) { 108 106 109 - $resource_phid = $resource->getPHID(); 110 - 111 107 $lease 112 108 ->setActivateWhenAcquired(true) 113 - ->needSlotLock("almanac.host.lease({$resource_phid})") 109 + ->needSlotLock($this->getLeaseSlotLock($resource)) 114 110 ->acquireOnResource($resource); 111 + } 112 + 113 + private function getLeaseSlotLock(DrydockResource $resource) { 114 + $resource_phid = $resource->getPHID(); 115 + return "almanac.host.lease({$resource_phid})"; 115 116 } 116 117 117 118 public function getType() {
+33
src/applications/drydock/blueprint/DrydockBlueprintImplementation.php
··· 67 67 DrydockResource $resource, 68 68 DrydockLease $lease); 69 69 70 + public function activateLease( 71 + DrydockBlueprint $blueprint, 72 + DrydockResource $resource, 73 + DrydockLease $lease) { 74 + throw new PhutilMethodNotImplementedException(); 75 + } 70 76 71 77 final public function releaseLease( 72 78 DrydockBlueprint $blueprint, ··· 198 204 DrydockBlueprint $blueprint, 199 205 DrydockLease $lease); 200 206 207 + public function activateResource( 208 + DrydockBlueprint $blueprint, 209 + DrydockResource $resource) { 210 + throw new PhutilMethodNotImplementedException(); 211 + } 201 212 202 213 /* -( Resource Interfaces )------------------------------------------------ */ 203 214 ··· 276 287 ->setStatus(DrydockResourceStatus::STATUS_PENDING) 277 288 ->setName($name); 278 289 290 + // Pre-allocate the resource PHID. 291 + $resource->setPHID($resource->generatePHID()); 292 + 279 293 $this->activeResource = $resource; 280 294 281 295 $this->log( ··· 284 298 get_class($this))); 285 299 286 300 return $resource; 301 + } 302 + 303 + protected function newLease(DrydockBlueprint $blueprint) { 304 + return id(new DrydockLease()); 305 + } 306 + 307 + protected function requireActiveLease(DrydockLease $lease) { 308 + $lease_status = $lease->getStatus(); 309 + 310 + switch ($lease_status) { 311 + case DrydockLeaseStatus::STATUS_ACQUIRED: 312 + // TODO: Temporary failure. 313 + throw new Exception(pht('Lease still activating.')); 314 + case DrydockLeaseStatus::STATUS_ACTIVE: 315 + return; 316 + default: 317 + // TODO: Permanent failure. 318 + throw new Exception(pht('Lease in bad state.')); 319 + } 287 320 } 288 321 289 322 private function pushActiveScope(
+154 -57
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
··· 17 17 18 18 public function canAnyBlueprintEverAllocateResourceForLease( 19 19 DrydockLease $lease) { 20 - // TODO: These checks are out of date. 21 20 return true; 22 21 } 23 22 24 23 public function canEverAllocateResourceForLease( 25 24 DrydockBlueprint $blueprint, 26 25 DrydockLease $lease) { 27 - // TODO: These checks are out of date. 28 26 return true; 29 27 } 30 28 31 29 public function canAllocateResourceForLease( 32 30 DrydockBlueprint $blueprint, 33 31 DrydockLease $lease) { 34 - // TODO: These checks are out of date. 35 32 return true; 36 33 } 37 34 ··· 39 36 DrydockBlueprint $blueprint, 40 37 DrydockResource $resource, 41 38 DrydockLease $lease) { 42 - // TODO: These checks are out of date. 39 + 40 + $have_phid = $resource->getAttribute('repositoryPHID'); 41 + $need_phid = $lease->getAttribute('repositoryPHID'); 43 42 44 - $resource_repo = $resource->getAttribute('repositoryID'); 45 - $lease_repo = $lease->getAttribute('repositoryID'); 43 + if ($need_phid !== $have_phid) { 44 + return false; 45 + } 46 + 47 + if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) { 48 + return false; 49 + } 46 50 47 - return ($resource_repo && $lease_repo && ($resource_repo == $lease_repo)); 51 + return true; 48 52 } 49 53 50 - public function allocateResource( 54 + public function acquireLease( 51 55 DrydockBlueprint $blueprint, 56 + DrydockResource $resource, 52 57 DrydockLease $lease) { 53 58 54 - $repository_id = $lease->getAttribute('repositoryID'); 55 - if (!$repository_id) { 56 - throw new Exception( 57 - pht( 58 - "Lease is missing required '%s' attribute.", 59 - 'repositoryID')); 60 - } 59 + $lease 60 + ->needSlotLock($this->getLeaseSlotLock($resource)) 61 + ->acquireOnResource($resource); 62 + } 61 63 62 - $repository = id(new PhabricatorRepositoryQuery()) 63 - ->setViewer(PhabricatorUser::getOmnipotentUser()) 64 - ->withIDs(array($repository_id)) 65 - ->executeOne(); 64 + private function getLeaseSlotLock(DrydockResource $resource) { 65 + $resource_phid = $resource->getPHID(); 66 + return "workingcopy.lease({$resource_phid})"; 67 + } 66 68 67 - if (!$repository) { 68 - throw new Exception( 69 - pht( 70 - "Repository '%s' does not exist!", 71 - $repository_id)); 72 - } 69 + public function allocateResource( 70 + DrydockBlueprint $blueprint, 71 + DrydockLease $lease) { 73 72 74 - switch ($repository->getVersionControlSystem()) { 75 - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: 76 - break; 77 - default: 78 - throw new Exception(pht('Unsupported VCS!')); 79 - } 73 + $repository_phid = $lease->getAttribute('repositoryPHID'); 74 + $repository = $this->loadRepository($repository_phid); 80 75 81 - // TODO: Policy stuff here too. 82 - $host_lease = id(new DrydockLease()) 76 + $resource = $this->newResourceTemplate( 77 + $blueprint, 78 + pht( 79 + 'Working Copy (%s)', 80 + $repository->getCallsign())); 81 + 82 + $resource_phid = $resource->getPHID(); 83 + 84 + $host_lease = $this->newLease($blueprint) 83 85 ->setResourceType('host') 84 - ->waitUntilActive(); 86 + ->setOwnerPHID($resource_phid) 87 + ->setAttribute('workingcopy.resourcePHID', $resource_phid) 88 + ->queueForActivation(); 89 + 90 + // TODO: Add some limits to the number of working copies we can have at 91 + // once? 92 + 93 + return $resource 94 + ->setAttribute('repositoryPHID', $repository->getPHID()) 95 + ->setAttribute('host.leasePHID', $host_lease->getPHID()) 96 + ->allocateResource(); 97 + } 85 98 86 - $path = $host_lease->getAttribute('path').$repository->getCallsign(); 99 + public function activateResource( 100 + DrydockBlueprint $blueprint, 101 + DrydockResource $resource) { 87 102 88 - $this->log( 89 - pht('Cloning %s into %s....', $repository->getCallsign(), $path)); 103 + $lease = $this->loadHostLease($resource); 104 + $this->requireActiveLease($lease); 90 105 91 - $cmd = $host_lease->getInterface('command'); 92 - $cmd->execx( 93 - 'git clone --origin origin %P %s', 94 - $repository->getRemoteURIEnvelope(), 95 - $path); 106 + $repository_phid = $resource->getAttribute('repositoryPHID'); 107 + $repository = $this->loadRepository($repository_phid); 108 + $repository_id = $repository->getID(); 109 + 110 + $command_type = DrydockCommandInterface::INTERFACE_TYPE; 111 + $interface = $lease->getInterface($command_type); 96 112 97 - $this->log(pht('Complete.')); 113 + // TODO: Make this configurable. 114 + $resource_id = $resource->getID(); 115 + $root = "/var/drydock/workingcopy-{$resource_id}"; 116 + $path = "{$root}/repo/{$repository_id}/"; 98 117 99 - $resource = $this->newResourceTemplate( 100 - $blueprint, 101 - pht( 102 - 'Working Copy (%s)', 103 - $repository->getCallsign())); 104 - $resource->setStatus(DrydockResourceStatus::STATUS_OPEN); 105 - $resource->setAttribute('lease.host', $host_lease->getID()); 106 - $resource->setAttribute('path', $path); 107 - $resource->setAttribute('repositoryID', $repository->getID()); 108 - $resource->save(); 118 + $interface->execx( 119 + 'git clone -- %s %s', 120 + (string)$repository->getCloneURIObject(), 121 + $path); 109 122 110 - return $resource; 123 + $resource 124 + ->setAttribute('workingcopy.root', $root) 125 + ->setAttribute('workingcopy.path', $path) 126 + ->activateResource(); 111 127 } 112 128 113 - public function acquireLease( 129 + public function activateLease( 114 130 DrydockBlueprint $blueprint, 115 131 DrydockResource $resource, 116 132 DrydockLease $lease) { 117 - return; 133 + 134 + $command_type = DrydockCommandInterface::INTERFACE_TYPE; 135 + $interface = $lease->getInterface($command_type); 136 + 137 + $cmd = array(); 138 + $arg = array(); 139 + 140 + $cmd[] = 'git clean -d --force'; 141 + $cmd[] = 'git reset --hard HEAD'; 142 + $cmd[] = 'git fetch'; 143 + 144 + $commit = $lease->getAttribute('commit'); 145 + $branch = $lease->getAttribute('branch'); 146 + 147 + if ($commit !== null) { 148 + $cmd[] = 'git reset --hard %s'; 149 + $arg[] = $commit; 150 + } else if ($branch !== null) { 151 + $cmd[] = 'git reset --hard %s'; 152 + $arg[] = $branch; 153 + } 154 + 155 + $cmd = implode(' && ', $cmd); 156 + $argv = array_merge(array($cmd), $arg); 157 + 158 + $result = call_user_func_array( 159 + array($interface, 'execx'), 160 + $argv); 161 + 162 + $lease->activateOnResource($resource); 118 163 } 119 164 120 165 public function getType() { ··· 126 171 DrydockResource $resource, 127 172 DrydockLease $lease, 128 173 $type) { 129 - // TODO: This blueprint doesn't work at all. 174 + 175 + switch ($type) { 176 + case DrydockCommandInterface::INTERFACE_TYPE: 177 + $host_lease = $this->loadHostLease($resource); 178 + $command_interface = $host_lease->getInterface($type); 179 + 180 + $path = $resource->getAttribute('workingcopy.path'); 181 + $command_interface->setWorkingDirectory($path); 182 + 183 + return $command_interface; 184 + } 130 185 } 186 + 187 + private function loadRepository($repository_phid) { 188 + $repository = id(new PhabricatorRepositoryQuery()) 189 + ->setViewer(PhabricatorUser::getOmnipotentUser()) 190 + ->withPHIDs(array($repository_phid)) 191 + ->executeOne(); 192 + if (!$repository) { 193 + // TODO: Permanent failure. 194 + throw new Exception( 195 + pht( 196 + 'Repository PHID "%s" does not exist.', 197 + $repository_phid)); 198 + } 199 + 200 + switch ($repository->getVersionControlSystem()) { 201 + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: 202 + break; 203 + default: 204 + // TODO: Permanent failure. 205 + throw new Exception(pht('Unsupported VCS!')); 206 + } 207 + 208 + return $repository; 209 + } 210 + 211 + private function loadHostLease(DrydockResource $resource) { 212 + $viewer = PhabricatorUser::getOmnipotentUser(); 213 + 214 + $lease_phid = $resource->getAttribute('host.leasePHID'); 215 + 216 + $lease = id(new DrydockLeaseQuery()) 217 + ->setViewer($viewer) 218 + ->withPHIDs(array($lease_phid)) 219 + ->executeOne(); 220 + if (!$lease) { 221 + // TODO: Permanent failure. 222 + throw new Exception(pht('Unable to load lease "%s".', $lease_phid)); 223 + } 224 + 225 + return $lease; 226 + } 227 + 131 228 132 229 }
+25
src/applications/drydock/exception/DrydockSlotLockException.php
··· 1 + <?php 2 + 3 + final class DrydockSlotLockException extends Exception { 4 + 5 + private $lockMap; 6 + 7 + public function __construct(array $locks) { 8 + $this->lockMap = $locks; 9 + 10 + if ($locks) { 11 + $lock_list = array(); 12 + foreach ($locks as $lock => $owner_phid) { 13 + $lock_list[] = pht('"%s" (owned by "%s")', $lock, $owner_phid); 14 + } 15 + $message = pht( 16 + 'Unable to acquire slot locks: %s.', 17 + implode(', ', $lock_list)); 18 + } else { 19 + $message = pht('Unable to acquire slot locks.'); 20 + } 21 + 22 + parent::__construct($message); 23 + } 24 + 25 + }
-2
src/applications/drydock/management/DrydockManagementLeaseWorkflow.php
··· 40 40 $attributes = $options->parse($attributes); 41 41 } 42 42 43 - PhabricatorWorker::setRunAllTasksInProcess(true); 44 - 45 43 $lease = id(new DrydockLease()) 46 44 ->setResourceType($resource_type); 47 45 if ($attributes) {
+22
src/applications/drydock/storage/DrydockBlueprint.php
··· 134 134 } 135 135 136 136 137 + /** 138 + * @task resource 139 + */ 140 + public function activateResource(DrydockResource $resource) { 141 + return $this->getImplementation()->activateResource( 142 + $this, 143 + $resource); 144 + } 145 + 137 146 /* -( Acquiring Leases )--------------------------------------------------- */ 138 147 139 148 ··· 157 166 DrydockResource $resource, 158 167 DrydockLease $lease) { 159 168 return $this->getImplementation()->acquireLease( 169 + $this, 170 + $resource, 171 + $lease); 172 + } 173 + 174 + 175 + /** 176 + * @task lease 177 + */ 178 + public function activateLease( 179 + DrydockResource $resource, 180 + DrydockLease $lease) { 181 + return $this->getImplementation()->activateLease( 160 182 $this, 161 183 $resource, 162 184 $lease);
+49 -4
src/applications/drydock/storage/DrydockLease.php
··· 14 14 private $resource = self::ATTACHABLE; 15 15 private $releaseOnDestruction; 16 16 private $isAcquired = false; 17 + private $isActivated = false; 17 18 private $activateWhenAcquired = false; 18 19 private $slotLocks = array(); 19 20 ··· 111 112 112 113 $task = PhabricatorWorker::scheduleTask( 113 114 'DrydockAllocatorWorker', 114 - $this->getID()); 115 + array( 116 + 'leasePHID' => $this->getPHID(), 117 + ), 118 + array( 119 + 'objectPHID' => $this->getPHID(), 120 + )); 115 121 116 122 // NOTE: Scheduling the task might execute it in-process, if we're running 117 123 // from a CLI script. Reload the lease to make sure we have the most ··· 229 235 if ($this->activateWhenAcquired) { 230 236 $new_status = DrydockLeaseStatus::STATUS_ACTIVE; 231 237 } else { 232 - $new_status = DrydockLeaseStatus::STATUS_PENDING; 238 + $new_status = DrydockLeaseStatus::STATUS_ACQUIRED; 233 239 } 234 240 235 - if ($new_status === DrydockLeaseStatus::STATUS_ACTIVE) { 236 - if ($resource->getStatus() === DrydockResourceStatus::STATUS_PENDING) { 241 + if ($new_status == DrydockLeaseStatus::STATUS_ACTIVE) { 242 + if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { 237 243 throw new Exception( 238 244 pht( 239 245 'Trying to acquire an active lease on a pending resource. '. ··· 261 267 262 268 public function isAcquiredLease() { 263 269 return $this->isAcquired; 270 + } 271 + 272 + public function activateOnResource(DrydockResource $resource) { 273 + $expect_status = DrydockLeaseStatus::STATUS_ACQUIRED; 274 + $actual_status = $this->getStatus(); 275 + if ($actual_status != $expect_status) { 276 + throw new Exception( 277 + pht( 278 + 'Trying to activate a lease which has the wrong status: status '. 279 + 'must be "%s", actually "%s".', 280 + $expect_status, 281 + $actual_status)); 282 + } 283 + 284 + if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { 285 + // TODO: Be stricter about this? 286 + throw new Exception( 287 + pht( 288 + 'Trying to activate a lease on a pending resource.')); 289 + } 290 + 291 + $this->openTransaction(); 292 + 293 + $this 294 + ->setStatus(DrydockLeaseStatus::STATUS_ACTIVE) 295 + ->save(); 296 + 297 + DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); 298 + $this->slotLocks = array(); 299 + 300 + $this->saveTransaction(); 301 + 302 + $this->isActivated = true; 303 + 304 + return $this; 305 + } 306 + 307 + public function isActivatedLease() { 308 + return $this->isActivated; 264 309 } 265 310 266 311
+40 -1
src/applications/drydock/storage/DrydockResource.php
··· 16 16 17 17 private $blueprint = self::ATTACHABLE; 18 18 private $isAllocated = false; 19 + private $isActivated = false; 19 20 private $activateWhenAllocated = false; 20 21 private $slotLocks = array(); 21 22 ··· 86 87 return $this; 87 88 } 88 89 89 - public function allocateResource($status) { 90 + public function allocateResource() { 90 91 if ($this->getID()) { 91 92 throw new Exception( 92 93 pht( ··· 129 130 130 131 public function isAllocatedResource() { 131 132 return $this->isAllocated; 133 + } 134 + 135 + public function activateResource() { 136 + if (!$this->getID()) { 137 + throw new Exception( 138 + pht( 139 + 'Trying to activate a resource which has not yet been persisted.')); 140 + } 141 + 142 + $expect_status = DrydockResourceStatus::STATUS_PENDING; 143 + $actual_status = $this->getStatus(); 144 + if ($actual_status != $expect_status) { 145 + throw new Exception( 146 + pht( 147 + 'Trying to activate a resource from the wrong status. Status must '. 148 + 'be "%s", actually "%s".', 149 + $expect_status, 150 + $actual_status)); 151 + } 152 + 153 + $this->openTransaction(); 154 + 155 + $this 156 + ->setStatus(DrydockResourceStatus::STATUS_OPEN) 157 + ->save(); 158 + 159 + DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks); 160 + $this->slotLocks = array(); 161 + 162 + $this->saveTransaction(); 163 + 164 + $this->isActivated = true; 165 + 166 + return $this; 167 + } 168 + 169 + public function isActivatedResource() { 170 + return $this->isActivated; 132 171 } 133 172 134 173 public function closeResource() {
+77 -9
src/applications/drydock/storage/DrydockSlotLock.php
··· 8 8 * machine. These optimistic "slot locks" provide a flexible way to do this 9 9 * sort of simple locking. 10 10 * 11 + * @task info Getting Lock Information 11 12 * @task lock Acquiring and Releasing Locks 12 13 */ 13 14 final class DrydockSlotLock extends DrydockDAO { ··· 35 36 ) + parent::getConfiguration(); 36 37 } 37 38 39 + 40 + /* -( Getting Lock Information )------------------------------------------- */ 41 + 42 + 43 + /** 44 + * Load all locks held by a particular owner. 45 + * 46 + * @param phid Owner PHID. 47 + * @return list<DrydockSlotLock> All held locks. 48 + * @task info 49 + */ 38 50 public static function loadLocks($owner_phid) { 39 51 return id(new DrydockSlotLock())->loadAllWhere( 40 52 'ownerPHID = %s', ··· 42 54 } 43 55 44 56 57 + /** 58 + * Test if a lock is currently free. 59 + * 60 + * @param string Lock key to test. 61 + * @return bool True if the lock is currently free. 62 + * @task info 63 + */ 64 + public static function isLockFree($lock) { 65 + return self::areLocksFree(array($lock)); 66 + } 67 + 68 + 69 + /** 70 + * Test if a list of locks are all currently free. 71 + * 72 + * @param list<string> List of lock keys to test. 73 + * @return bool True if all locks are currently free. 74 + * @task info 75 + */ 76 + public static function areLocksFree(array $locks) { 77 + $lock_map = self::loadHeldLocks($locks); 78 + return !$lock_map; 79 + } 80 + 81 + 82 + /** 83 + * Load named locks. 84 + * 85 + * @param list<string> List of lock keys to load. 86 + * @return list<DrydockSlotLock> List of held locks. 87 + * @task info 88 + */ 89 + public static function loadHeldLocks(array $locks) { 90 + if (!$locks) { 91 + return array(); 92 + } 93 + 94 + $table = new DrydockSlotLock(); 95 + $conn_r = $table->establishConnection('r'); 96 + 97 + $indexes = array(); 98 + foreach ($locks as $lock) { 99 + $indexes[] = PhabricatorHash::digestForIndex($lock); 100 + } 101 + 102 + return id(new DrydockSlotLock())->loadAllWhere( 103 + 'lockIndex IN (%Ls)', 104 + $indexes); 105 + } 106 + 107 + 45 108 /* -( Acquiring and Releasing Locks )-------------------------------------- */ 46 109 47 110 ··· 74 137 $lock); 75 138 } 76 139 77 - // TODO: These exceptions are pretty tricky to read. It would be good to 78 - // figure out which locks could not be acquired and try to improve the 79 - // exception to make debugging easier. 80 - 81 - queryfx( 82 - $conn_w, 83 - 'INSERT INTO %T (ownerPHID, lockIndex, lockKey) VALUES %Q', 84 - $table->getTableName(), 85 - implode(', ', $sql)); 140 + try { 141 + queryfx( 142 + $conn_w, 143 + 'INSERT INTO %T (ownerPHID, lockIndex, lockKey) VALUES %Q', 144 + $table->getTableName(), 145 + implode(', ', $sql)); 146 + } catch (AphrontDuplicateKeyQueryException $ex) { 147 + // Try to improve the readability of the exception. We might miss on 148 + // this query if the lock has already been released, but most of the 149 + // time we should be able to figure out which locks are already held. 150 + $held = self::loadHeldLocks($locks); 151 + $held = mpull($held, 'getOwnerPHID', 'getLockKey'); 152 + throw new DrydockSlotLockException($held); 153 + } 86 154 } 87 155 88 156
+31 -25
src/applications/drydock/worker/DrydockAllocatorWorker.php
··· 5 5 * @task resource Managing Resources 6 6 * @task lease Managing Leases 7 7 */ 8 - final class DrydockAllocatorWorker extends PhabricatorWorker { 8 + final class DrydockAllocatorWorker extends DrydockWorker { 9 9 10 - private function getViewer() { 11 - return PhabricatorUser::getOmnipotentUser(); 12 - } 13 - 14 - private function loadLease() { 15 - $viewer = $this->getViewer(); 16 - 17 - // TODO: Make the task data a dictionary like every other worker, and 18 - // probably make this a PHID. 19 - $lease_id = $this->getTaskData(); 10 + protected function doWork() { 11 + $lease_phid = $this->getTaskDataValue('leasePHID'); 12 + $lease = $this->loadLease($lease_phid); 20 13 21 - $lease = id(new DrydockLeaseQuery()) 22 - ->setViewer($viewer) 23 - ->withIDs(array($lease_id)) 24 - ->executeOne(); 25 - if (!$lease) { 26 - throw new PhabricatorWorkerPermanentFailureException( 27 - pht('No such lease "%s"!', $lease_id)); 28 - } 29 - 30 - return $lease; 31 - } 32 - 33 - protected function doWork() { 34 - $lease = $this->loadLease(); 35 14 $this->allocateAndAcquireLease($lease); 36 15 } 37 16 ··· 351 330 DrydockLease $lease) { 352 331 $resource = $blueprint->allocateResource($lease); 353 332 $this->validateAllocatedResource($blueprint, $resource, $lease); 333 + 334 + // If this resource was allocated as a pending resource, queue a task to 335 + // activate it. 336 + if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { 337 + PhabricatorWorker::scheduleTask( 338 + 'DrydockResourceWorker', 339 + array( 340 + 'resourcePHID' => $resource->getPHID(), 341 + ), 342 + array( 343 + 'objectPHID' => $resource->getPHID(), 344 + )); 345 + } 346 + 354 347 return $resource; 355 348 } 356 349 ··· 429 422 $blueprint->acquireLease($resource, $lease); 430 423 431 424 $this->validateAcquiredLease($blueprint, $resource, $lease); 425 + 426 + // If this lease has been acquired but not activated, queue a task to 427 + // activate it. 428 + if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) { 429 + PhabricatorWorker::scheduleTask( 430 + 'DrydockLeaseWorker', 431 + array( 432 + 'leasePHID' => $lease->getPHID(), 433 + ), 434 + array( 435 + 'objectPHID' => $lease->getPHID(), 436 + )); 437 + } 432 438 } 433 439 434 440
+81
src/applications/drydock/worker/DrydockLeaseWorker.php
··· 1 + <?php 2 + 3 + final class DrydockLeaseWorker extends DrydockWorker { 4 + 5 + protected function doWork() { 6 + $lease_phid = $this->getTaskDataValue('leasePHID'); 7 + $lease = $this->loadLease($lease_phid); 8 + 9 + $this->activateLease($lease); 10 + } 11 + 12 + 13 + private function activateLease(DrydockLease $lease) { 14 + $actual_status = $lease->getStatus(); 15 + 16 + if ($actual_status != DrydockLeaseStatus::STATUS_ACQUIRED) { 17 + throw new PhabricatorWorkerPermanentFailureException( 18 + pht( 19 + 'Trying to activate lease from wrong status ("%s").', 20 + $actual_status)); 21 + } 22 + 23 + $resource_id = $lease->getResourceID(); 24 + 25 + $resource = id(new DrydockResourceQuery()) 26 + ->setViewer($this->getViewer()) 27 + ->withIDs(array($resource_id)) 28 + ->executeOne(); 29 + if (!$resource) { 30 + throw new PhabricatorWorkerPermanentFailureException( 31 + pht( 32 + 'Trying to activate lease on invalid resource ("%s").', 33 + $resource_id)); 34 + } 35 + 36 + $resource_status = $resource->getStatus(); 37 + 38 + if ($resource_status == DrydockResourceStatus::STATUS_PENDING) { 39 + // TODO: This is explicitly a temporary failure -- we are waiting for 40 + // the resource to come up. 41 + throw new Exception(pht('Resource still activating.')); 42 + } 43 + 44 + if ($resource_status != DrydockResourceStatus::STATUS_OPEN) { 45 + throw new PhabricatorWorkerPermanentFailureException( 46 + pht( 47 + 'Trying to activate lease on a dead resource (in status "%s").', 48 + $resource_status)); 49 + } 50 + 51 + // NOTE: We can race resource destruction here. Between the time we 52 + // performed the read above and now, the resource might have closed, so 53 + // we may activate leases on dead resources. At least for now, this seems 54 + // fine: a resource dying right before we activate a lease on it should not 55 + // be distinguisahble from a resource dying right after we activate a lease 56 + // on it. We end up with an active lease on a dead resource either way, and 57 + // can not prevent resources dying from lightning strikes. 58 + 59 + $blueprint = $resource->getBlueprint(); 60 + $blueprint->activateLease($resource, $lease); 61 + $this->validateActivatedLease($blueprint, $resource, $lease); 62 + } 63 + 64 + private function validateActivatedLease( 65 + DrydockBlueprint $blueprint, 66 + DrydockResource $resource, 67 + DrydockLease $lease) { 68 + 69 + if (!$lease->isActivatedLease()) { 70 + throw new Exception( 71 + pht( 72 + 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 73 + 'returned from "%s" without activating a lease.', 74 + $blueprint->getBlueprintName(), 75 + $blueprint->getClassName(), 76 + 'acquireLease()')); 77 + } 78 + 79 + } 80 + 81 + }
+45
src/applications/drydock/worker/DrydockResourceWorker.php
··· 1 + <?php 2 + 3 + final class DrydockResourceWorker extends DrydockWorker { 4 + 5 + protected function doWork() { 6 + $resource_phid = $this->getTaskDataValue('resourcePHID'); 7 + $resource = $this->loadResource($resource_phid); 8 + 9 + $this->activateResource($resource); 10 + } 11 + 12 + 13 + private function activateResource(DrydockResource $resource) { 14 + $resource_status = $resource->getStatus(); 15 + 16 + if ($resource_status != DrydockResourceStatus::STATUS_PENDING) { 17 + throw new PhabricatorWorkerPermanentFailureException( 18 + pht( 19 + 'Trying to activate resource from wrong status ("%s").', 20 + $resource_status)); 21 + } 22 + 23 + $blueprint = $resource->getBlueprint(); 24 + $blueprint->activateResource($resource); 25 + $this->validateActivatedResource($blueprint, $resource); 26 + } 27 + 28 + 29 + private function validateActivatedResource( 30 + DrydockBlueprint $blueprint, 31 + DrydockResource $resource) { 32 + 33 + if (!$resource->isActivatedResource()) { 34 + throw new Exception( 35 + pht( 36 + 'Blueprint "%s" (of type "%s") is not properly implemented: %s '. 37 + 'must actually allocate the resource it returns.', 38 + $blueprint->getBlueprintName(), 39 + $blueprint->getClassName(), 40 + 'allocateResource()')); 41 + } 42 + 43 + } 44 + 45 + }
+39
src/applications/drydock/worker/DrydockWorker.php
··· 1 + <?php 2 + 3 + abstract class DrydockWorker extends PhabricatorWorker { 4 + 5 + protected function getViewer() { 6 + return PhabricatorUser::getOmnipotentUser(); 7 + } 8 + 9 + protected function loadLease($lease_phid) { 10 + $viewer = $this->getViewer(); 11 + 12 + $lease = id(new DrydockLeaseQuery()) 13 + ->setViewer($viewer) 14 + ->withPHIDs(array($lease_phid)) 15 + ->executeOne(); 16 + if (!$lease) { 17 + throw new PhabricatorWorkerPermanentFailureException( 18 + pht('No such lease "%s"!', $lease_phid)); 19 + } 20 + 21 + return $lease; 22 + } 23 + 24 + protected function loadResource($resource_phid) { 25 + $viewer = $this->getViewer(); 26 + 27 + $resource = id(new DrydockResourceQuery()) 28 + ->setViewer($viewer) 29 + ->withPHIDs(array($resource_phid)) 30 + ->executeOne(); 31 + if (!$resource) { 32 + throw new PhabricatorWorkerPermanentFailureException( 33 + pht('No such resource "%s"!', $resource_phid)); 34 + } 35 + 36 + return $resource; 37 + } 38 + 39 + }
+13 -20
src/infrastructure/daemon/workers/PhabricatorWorker.php
··· 87 87 return $this->data; 88 88 } 89 89 90 + final protected function getTaskDataValue($key, $default = null) { 91 + $data = $this->getTaskData(); 92 + if (!is_array($data)) { 93 + throw new PhabricatorWorkerPermanentFailureException( 94 + pht('Expected task data to be a dictionary.')); 95 + } 96 + return idx($data, $key, $default); 97 + } 98 + 90 99 final public function executeTask() { 91 100 $this->doWork(); 92 101 } ··· 149 158 150 159 151 160 /** 152 - * Wait for tasks to complete. If tasks are not leased by other workers, they 153 - * will be executed in this process while waiting. 161 + * Wait for tasks to complete. 154 162 * 155 163 * @param list<int> List of queued task IDs to wait for. 156 164 * @return void ··· 178 186 break; 179 187 } 180 188 181 - $tasks = id(new PhabricatorWorkerLeaseQuery()) 182 - ->withIDs($waiting) 183 - ->setLimit(1) 184 - ->execute(); 185 - 186 - if (!$tasks) { 187 - // We were not successful in leasing anything. Sleep for a bit and 188 - // see if we have better luck later. 189 - sleep(1); 190 - continue; 191 - } 192 - 193 - $task = head($tasks)->executeTask(); 194 - 195 - $ex = $task->getExecutionException(); 196 - if ($ex) { 197 - throw $ex; 198 - } 189 + // We were not successful in leasing anything. Sleep for a bit and 190 + // see if we have better luck later. 191 + sleep(1); 199 192 } 200 193 201 194 $tasks = id(new PhabricatorWorkerArchiveTaskQuery())