Merge the DrydockLease workers into a single worker

-6

src/__phutil_library_map__.php

··· 796 796 'DoorkeeperSchemaSpec' => 'applications/doorkeeper/storage/DoorkeeperSchemaSpec.php', 797 797 'DoorkeeperTagView' => 'applications/doorkeeper/view/DoorkeeperTagView.php', 798 798 'DoorkeeperTagsController' => 'applications/doorkeeper/controller/DoorkeeperTagsController.php', 799 - 'DrydockAllocatorWorker' => 'applications/drydock/worker/DrydockAllocatorWorker.php', 800 799 'DrydockAlmanacServiceHostBlueprintImplementation' => 'applications/drydock/blueprint/DrydockAlmanacServiceHostBlueprintImplementation.php', 801 800 'DrydockApacheWebrootInterface' => 'applications/drydock/interface/webroot/DrydockApacheWebrootInterface.php', 802 801 'DrydockBlueprint' => 'applications/drydock/storage/DrydockBlueprint.php', ··· 834 833 'DrydockLeaseActivatedLogType' => 'applications/drydock/logtype/DrydockLeaseActivatedLogType.php', 835 834 'DrydockLeaseController' => 'applications/drydock/controller/DrydockLeaseController.php', 836 835 'DrydockLeaseDatasource' => 'applications/drydock/typeahead/DrydockLeaseDatasource.php', 837 - 'DrydockLeaseDestroyWorker' => 'applications/drydock/worker/DrydockLeaseDestroyWorker.php', 838 836 'DrydockLeaseDestroyedLogType' => 'applications/drydock/logtype/DrydockLeaseDestroyedLogType.php', 839 837 'DrydockLeaseListController' => 'applications/drydock/controller/DrydockLeaseListController.php', 840 838 'DrydockLeaseListView' => 'applications/drydock/view/DrydockLeaseListView.php', ··· 847 845 'DrydockLeaseStatus' => 'applications/drydock/constants/DrydockLeaseStatus.php', 848 846 'DrydockLeaseUpdateWorker' => 'applications/drydock/worker/DrydockLeaseUpdateWorker.php', 849 847 'DrydockLeaseViewController' => 'applications/drydock/controller/DrydockLeaseViewController.php', 850 - 'DrydockLeaseWorker' => 'applications/drydock/worker/DrydockLeaseWorker.php', 851 848 'DrydockLog' => 'applications/drydock/storage/DrydockLog.php', 852 849 'DrydockLogController' => 'applications/drydock/controller/DrydockLogController.php', 853 850 'DrydockLogGarbageCollector' => 'applications/drydock/garbagecollector/DrydockLogGarbageCollector.php', ··· 4525 4522 'DoorkeeperSchemaSpec' => 'PhabricatorConfigSchemaSpec', 4526 4523 'DoorkeeperTagView' => 'AphrontView', 4527 4524 'DoorkeeperTagsController' => 'PhabricatorController', 4528 - 'DrydockAllocatorWorker' => 'DrydockWorker', 4529 4525 'DrydockAlmanacServiceHostBlueprintImplementation' => 'DrydockBlueprintImplementation', 4530 4526 'DrydockApacheWebrootInterface' => 'DrydockWebrootInterface', 4531 4527 'DrydockBlueprint' => array( ··· 4577 4573 'DrydockLeaseActivatedLogType' => 'DrydockLogType', 4578 4574 'DrydockLeaseController' => 'DrydockController', 4579 4575 'DrydockLeaseDatasource' => 'PhabricatorTypeaheadDatasource', 4580 - 'DrydockLeaseDestroyWorker' => 'DrydockWorker', 4581 4576 'DrydockLeaseDestroyedLogType' => 'DrydockLogType', 4582 4577 'DrydockLeaseListController' => 'DrydockLeaseController', 4583 4578 'DrydockLeaseListView' => 'AphrontView', ··· 4590 4585 'DrydockLeaseStatus' => 'DrydockConstants', 4591 4586 'DrydockLeaseUpdateWorker' => 'DrydockWorker', 4592 4587 'DrydockLeaseViewController' => 'DrydockLeaseController', 4593 - 'DrydockLeaseWorker' => 'DrydockWorker', 4594 4588 'DrydockLog' => array( 4595 4589 'DrydockDAO', 4596 4590 'PhabricatorPolicyInterface',

+6 -12

src/applications/drydock/storage/DrydockLease.php

··· 135 135 ->setStatus(DrydockLeaseStatus::STATUS_PENDING) 136 136 ->save(); 137 137 138 - $task = PhabricatorWorker::scheduleTask( 139 - 'DrydockAllocatorWorker', 140 - array( 141 - 'leasePHID' => $this->getPHID(), 142 - ), 143 - array( 144 - 'objectPHID' => $this->getPHID(), 145 - )); 138 + $this->scheduleUpdate(); 146 139 147 140 $this->logEvent(DrydockLeaseQueuedLogType::LOGCONST); 148 141 ··· 321 314 } 322 315 } 323 316 324 - public function canUpdate() { 317 + public function canReceiveCommands() { 325 318 switch ($this->getStatus()) { 326 - case DrydockLeaseStatus::STATUS_ACTIVE: 327 - return true; 319 + case DrydockLeaseStatus::STATUS_RELEASED: 320 + case DrydockLeaseStatus::STATUS_DESTROYED: 321 + return false; 328 322 default: 329 - return false; 323 + return true; 330 324 } 331 325 } 332 326

-479

src/applications/drydock/worker/DrydockAllocatorWorker.php

··· 1 - <?php 2 - 3 - /** 4 - * @task allocate Allocator 5 - * @task resource Managing Resources 6 - * @task lease Managing Leases 7 - */ 8 - final class DrydockAllocatorWorker extends DrydockWorker { 9 - 10 - protected function doWork() { 11 - $lease_phid = $this->getTaskDataValue('leasePHID'); 12 - $lease = $this->loadLease($lease_phid); 13 - 14 - $this->allocateAndAcquireLease($lease); 15 - } 16 - 17 - 18 - /* -( Allocator )---------------------------------------------------------- */ 19 - 20 - 21 - /** 22 - * Find or build a resource which can satisfy a given lease request, then 23 - * acquire the lease. 24 - * 25 - * @param DrydockLease Requested lease. 26 - * @return void 27 - * @task allocator 28 - */ 29 - private function allocateAndAcquireLease(DrydockLease $lease) { 30 - $blueprints = $this->loadBlueprintsForAllocatingLease($lease); 31 - 32 - // If we get nothing back, that means no blueprint is defined which can 33 - // ever build the requested resource. This is a permanent failure, since 34 - // we don't expect to succeed no matter how many times we try. 35 - if (!$blueprints) { 36 - $lease 37 - ->setStatus(DrydockLeaseStatus::STATUS_BROKEN) 38 - ->save(); 39 - throw new PhabricatorWorkerPermanentFailureException( 40 - pht( 41 - 'No active Drydock blueprint exists which can ever allocate a '. 42 - 'resource for lease "%s".', 43 - $lease->getPHID())); 44 - } 45 - 46 - // First, try to find a suitable open resource which we can acquire a new 47 - // lease on. 48 - $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease); 49 - 50 - // If no resources exist yet, see if we can build one. 51 - if (!$resources) { 52 - $usable_blueprints = $this->removeOverallocatedBlueprints( 53 - $blueprints, 54 - $lease); 55 - 56 - // If we get nothing back here, some blueprint claims it can eventually 57 - // satisfy the lease, just not right now. This is a temporary failure, 58 - // and we expect allocation to succeed eventually. 59 - if (!$blueprints) { 60 - // TODO: More formal temporary failure here. We should retry this 61 - // "soon" but not "immediately". 62 - throw new Exception( 63 - pht('No blueprints have space to allocate a resource right now.')); 64 - } 65 - 66 - $usable_blueprints = $this->rankBlueprints($blueprints, $lease); 67 - 68 - $exceptions = array(); 69 - foreach ($usable_blueprints as $blueprint) { 70 - try { 71 - $resources[] = $this->allocateResource($blueprint, $lease); 72 - 73 - // Bail after allocating one resource, we don't need any more than 74 - // this. 75 - break; 76 - } catch (Exception $ex) { 77 - $exceptions[] = $ex; 78 - } 79 - } 80 - 81 - if (!$resources) { 82 - // TODO: We should distinguish between temporary and permament failures 83 - // here. If any blueprint failed temporarily, retry "soon". If none 84 - // of these failures were temporary, maybe this should be a permanent 85 - // failure? 86 - throw new PhutilAggregateException( 87 - pht( 88 - 'All blueprints failed to allocate a suitable new resource when '. 89 - 'trying to allocate lease "%s".', 90 - $lease->getPHID()), 91 - $exceptions); 92 - } 93 - 94 - // NOTE: We have not acquired the lease yet, so it is possible that the 95 - // resource we just built will be snatched up by some other lease before 96 - // we can. This is not problematic: we'll retry a little later and should 97 - // suceed eventually. 98 - } 99 - 100 - $resources = $this->rankResources($resources, $lease); 101 - 102 - $exceptions = array(); 103 - $allocated = false; 104 - foreach ($resources as $resource) { 105 - try { 106 - $this->acquireLease($resource, $lease); 107 - $allocated = true; 108 - break; 109 - } catch (Exception $ex) { 110 - $exceptions[] = $ex; 111 - } 112 - } 113 - 114 - if (!$allocated) { 115 - // TODO: We should distinguish between temporary and permanent failures 116 - // here. If any failures were temporary (specifically, failed to acquire 117 - // locks) 118 - 119 - throw new PhutilAggregateException( 120 - pht( 121 - 'Unable to acquire lease "%s" on any resouce.', 122 - $lease->getPHID()), 123 - $exceptions); 124 - } 125 - } 126 - 127 - 128 - /** 129 - * Get all the @{class:DrydockBlueprintImplementation}s which can possibly 130 - * build a resource to satisfy a lease. 131 - * 132 - * This method returns blueprints which might, at some time, be able to 133 - * build a resource which can satisfy the lease. They may not be able to 134 - * build that resource right now. 135 - * 136 - * @param DrydockLease Requested lease. 137 - * @return list<DrydockBlueprintImplementation> List of qualifying blueprint 138 - * implementations. 139 - * @task allocator 140 - */ 141 - private function loadBlueprintImplementationsForAllocatingLease( 142 - DrydockLease $lease) { 143 - 144 - $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations(); 145 - 146 - $keep = array(); 147 - foreach ($impls as $key => $impl) { 148 - // Don't use disabled blueprint types. 149 - if (!$impl->isEnabled()) { 150 - continue; 151 - } 152 - 153 - // Don't use blueprint types which can't allocate the correct kind of 154 - // resource. 155 - if ($impl->getType() != $lease->getResourceType()) { 156 - continue; 157 - } 158 - 159 - if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) { 160 - continue; 161 - } 162 - 163 - $keep[$key] = $impl; 164 - } 165 - 166 - return $keep; 167 - } 168 - 169 - 170 - /** 171 - * Get all the concrete @{class:DrydockBlueprint}s which can possibly 172 - * build a resource to satisfy a lease. 173 - * 174 - * @param DrydockLease Requested lease. 175 - * @return list<DrydockBlueprint> List of qualifying blueprints. 176 - * @task allocator 177 - */ 178 - private function loadBlueprintsForAllocatingLease( 179 - DrydockLease $lease) { 180 - $viewer = $this->getViewer(); 181 - 182 - $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease); 183 - if (!$impls) { 184 - return array(); 185 - } 186 - 187 - $blueprints = id(new DrydockBlueprintQuery()) 188 - ->setViewer($viewer) 189 - ->withBlueprintClasses(array_keys($impls)) 190 - ->withDisabled(false) 191 - ->execute(); 192 - 193 - $keep = array(); 194 - foreach ($blueprints as $key => $blueprint) { 195 - if (!$blueprint->canEverAllocateResourceForLease($lease)) { 196 - continue; 197 - } 198 - 199 - $keep[$key] = $blueprint; 200 - } 201 - 202 - return $keep; 203 - } 204 - 205 - 206 - /** 207 - * Load a list of all resources which a given lease can possibly be 208 - * allocated against. 209 - * 210 - * @param list<DrydockBlueprint> Blueprints which may produce suitable 211 - * resources. 212 - * @param DrydockLease Requested lease. 213 - * @return list<DrydockResource> Resources which may be able to allocate 214 - * the lease. 215 - * @task allocator 216 - */ 217 - private function loadResourcesForAllocatingLease( 218 - array $blueprints, 219 - DrydockLease $lease) { 220 - assert_instances_of($blueprints, 'DrydockBlueprint'); 221 - $viewer = $this->getViewer(); 222 - 223 - $resources = id(new DrydockResourceQuery()) 224 - ->setViewer($viewer) 225 - ->withBlueprintPHIDs(mpull($blueprints, 'getPHID')) 226 - ->withTypes(array($lease->getResourceType())) 227 - ->withStatuses( 228 - array( 229 - DrydockResourceStatus::STATUS_PENDING, 230 - DrydockResourceStatus::STATUS_ACTIVE, 231 - )) 232 - ->execute(); 233 - 234 - $keep = array(); 235 - foreach ($resources as $key => $resource) { 236 - $blueprint = $resource->getBlueprint(); 237 - 238 - if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) { 239 - continue; 240 - } 241 - 242 - $keep[$key] = $resource; 243 - } 244 - 245 - return $keep; 246 - } 247 - 248 - 249 - /** 250 - * Remove blueprints which are too heavily allocated to build a resource for 251 - * a lease from a list of blueprints. 252 - * 253 - * @param list<DrydockBlueprint> List of blueprints. 254 - * @return list<DrydockBlueprint> List with blueprints that can not allocate 255 - * a resource for the lease right now removed. 256 - * @task allocator 257 - */ 258 - private function removeOverallocatedBlueprints( 259 - array $blueprints, 260 - DrydockLease $lease) { 261 - assert_instances_of($blueprints, 'DrydockBlueprint'); 262 - 263 - $keep = array(); 264 - foreach ($blueprints as $key => $blueprint) { 265 - if (!$blueprint->canAllocateResourceForLease($lease)) { 266 - continue; 267 - } 268 - 269 - $keep[$key] = $blueprint; 270 - } 271 - 272 - return $keep; 273 - } 274 - 275 - 276 - /** 277 - * Rank blueprints by suitability for building a new resource for a 278 - * particular lease. 279 - * 280 - * @param list<DrydockBlueprint> List of blueprints. 281 - * @param DrydockLease Requested lease. 282 - * @return list<DrydockBlueprint> Ranked list of blueprints. 283 - * @task allocator 284 - */ 285 - private function rankBlueprints(array $blueprints, DrydockLease $lease) { 286 - assert_instances_of($blueprints, 'DrydockBlueprint'); 287 - 288 - // TODO: Implement improvements to this ranking algorithm if they become 289 - // available. 290 - shuffle($blueprints); 291 - 292 - return $blueprints; 293 - } 294 - 295 - 296 - /** 297 - * Rank resources by suitability for allocating a particular lease. 298 - * 299 - * @param list<DrydockResource> List of resources. 300 - * @param DrydockLease Requested lease. 301 - * @return list<DrydockResource> Ranked list of resources. 302 - * @task allocator 303 - */ 304 - private function rankResources(array $resources, DrydockLease $lease) { 305 - assert_instances_of($resources, 'DrydockResource'); 306 - 307 - // TODO: Implement improvements to this ranking algorithm if they become 308 - // available. 309 - shuffle($resources); 310 - 311 - return $resources; 312 - } 313 - 314 - 315 - /* -( Managing Resources )------------------------------------------------- */ 316 - 317 - 318 - /** 319 - * Perform an actual resource allocation with a particular blueprint. 320 - * 321 - * @param DrydockBlueprint The blueprint to allocate a resource from. 322 - * @param DrydockLease Requested lease. 323 - * @return DrydockResource Allocated resource. 324 - * @task resource 325 - */ 326 - private function allocateResource( 327 - DrydockBlueprint $blueprint, 328 - DrydockLease $lease) { 329 - $resource = $blueprint->allocateResource($lease); 330 - $this->validateAllocatedResource($blueprint, $resource, $lease); 331 - 332 - // If this resource was allocated as a pending resource, queue a task to 333 - // activate it. 334 - if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { 335 - PhabricatorWorker::scheduleTask( 336 - 'DrydockResourceUpdateWorker', 337 - array( 338 - 'resourcePHID' => $resource->getPHID(), 339 - ), 340 - array( 341 - 'objectPHID' => $resource->getPHID(), 342 - )); 343 - } 344 - 345 - return $resource; 346 - } 347 - 348 - 349 - /** 350 - * Check that the resource a blueprint allocated is roughly the sort of 351 - * object we expect. 352 - * 353 - * @param DrydockBlueprint Blueprint which built the resource. 354 - * @param wild Thing which the blueprint claims is a valid resource. 355 - * @param DrydockLease Lease the resource was allocated for. 356 - * @return void 357 - * @task resource 358 - */ 359 - private function validateAllocatedResource( 360 - DrydockBlueprint $blueprint, 361 - $resource, 362 - DrydockLease $lease) { 363 - 364 - if (!($resource instanceof DrydockResource)) { 365 - throw new Exception( 366 - pht( 367 - 'Blueprint "%s" (of type "%s") is not properly implemented: %s must '. 368 - 'return an object of type %s or throw, but returned something else.', 369 - $blueprint->getBlueprintName(), 370 - $blueprint->getClassName(), 371 - 'allocateResource()', 372 - 'DrydockResource')); 373 - } 374 - 375 - if (!$resource->isAllocatedResource()) { 376 - throw new Exception( 377 - pht( 378 - 'Blueprint "%s" (of type "%s") is not properly implemented: %s '. 379 - 'must actually allocate the resource it returns.', 380 - $blueprint->getBlueprintName(), 381 - $blueprint->getClassName(), 382 - 'allocateResource()')); 383 - } 384 - 385 - $resource_type = $resource->getType(); 386 - $lease_type = $lease->getResourceType(); 387 - 388 - if ($resource_type !== $lease_type) { 389 - // TODO: Destroy the resource here? 390 - 391 - throw new Exception( 392 - pht( 393 - 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 394 - 'built a resource of type "%s" to satisfy a lease requesting a '. 395 - 'resource of type "%s".', 396 - $blueprint->getBlueprintName(), 397 - $blueprint->getClassName(), 398 - $resource_type, 399 - $lease_type)); 400 - } 401 - } 402 - 403 - 404 - /* -( Managing Leases )---------------------------------------------------- */ 405 - 406 - 407 - /** 408 - * Perform an actual lease acquisition on a particular resource. 409 - * 410 - * @param DrydockResource Resource to acquire a lease on. 411 - * @param DrydockLease Lease to acquire. 412 - * @return void 413 - * @task lease 414 - */ 415 - private function acquireLease( 416 - DrydockResource $resource, 417 - DrydockLease $lease) { 418 - 419 - $blueprint = $resource->getBlueprint(); 420 - $blueprint->acquireLease($resource, $lease); 421 - 422 - $this->validateAcquiredLease($blueprint, $resource, $lease); 423 - 424 - // If this lease has been acquired but not activated, queue a task to 425 - // activate it. 426 - if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) { 427 - PhabricatorWorker::scheduleTask( 428 - 'DrydockLeaseWorker', 429 - array( 430 - 'leasePHID' => $lease->getPHID(), 431 - ), 432 - array( 433 - 'objectPHID' => $lease->getPHID(), 434 - )); 435 - } 436 - } 437 - 438 - 439 - /** 440 - * Make sure that a lease was really acquired properly. 441 - * 442 - * @param DrydockBlueprint Blueprint which created the resource. 443 - * @param DrydockResource Resource which was acquired. 444 - * @param DrydockLease The lease which was supposedly acquired. 445 - * @return void 446 - * @task lease 447 - */ 448 - private function validateAcquiredLease( 449 - DrydockBlueprint $blueprint, 450 - DrydockResource $resource, 451 - DrydockLease $lease) { 452 - 453 - if (!$lease->isAcquiredLease()) { 454 - throw new Exception( 455 - pht( 456 - 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 457 - 'returned from "%s" without acquiring a lease.', 458 - $blueprint->getBlueprintName(), 459 - $blueprint->getClassName(), 460 - 'acquireLease()')); 461 - } 462 - 463 - $lease_phid = $lease->getResourcePHID(); 464 - $resource_phid = $resource->getPHID(); 465 - 466 - if ($lease_phid !== $resource_phid) { 467 - // TODO: Destroy the lease? 468 - throw new Exception( 469 - pht( 470 - 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 471 - 'returned from "%s" with a lease acquired on the wrong resource.', 472 - $blueprint->getBlueprintName(), 473 - $blueprint->getClassName(), 474 - 'acquireLease()')); 475 - } 476 - } 477 - 478 - 479 - }

-39

src/applications/drydock/worker/DrydockLeaseDestroyWorker.php

··· 1 - <?php 2 - 3 - final class DrydockLeaseDestroyWorker extends DrydockWorker { 4 - 5 - protected function doWork() { 6 - $lease_phid = $this->getTaskDataValue('leasePHID'); 7 - $lease = $this->loadLease($lease_phid); 8 - $this->destroyLease($lease); 9 - } 10 - 11 - private function destroyLease(DrydockLease $lease) { 12 - $status = $lease->getStatus(); 13 - 14 - switch ($status) { 15 - case DrydockLeaseStatus::STATUS_RELEASED: 16 - case DrydockLeaseStatus::STATUS_BROKEN: 17 - break; 18 - default: 19 - throw new PhabricatorWorkerPermanentFailureException( 20 - pht( 21 - 'Unable to destroy lease ("%s"), lease has the wrong '. 22 - 'status ("%s").', 23 - $lease->getPHID(), 24 - $status)); 25 - } 26 - 27 - $resource = $lease->getResource(); 28 - $blueprint = $resource->getBlueprint(); 29 - 30 - $blueprint->destroyLease($resource, $lease); 31 - 32 - $lease 33 - ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED) 34 - ->save(); 35 - 36 - $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST); 37 - } 38 - 39 - }

+602 -15

src/applications/drydock/worker/DrydockLeaseUpdateWorker.php

··· 1 1 <?php 2 2 3 + /** 4 + * @task update Updating Leases 5 + * @task command Processing Commands 6 + * @task allocator Drydock Allocator 7 + * @task acquire Acquiring Leases 8 + * @task activate Activating Leases 9 + * @task release Releasing Leases 10 + * @task destroy Destroying Leases 11 + */ 3 12 final class DrydockLeaseUpdateWorker extends DrydockWorker { 4 13 5 14 protected function doWork() { ··· 22 31 $lock->unlock(); 23 32 } 24 33 34 + 35 + /* -( Updating Leases )---------------------------------------------------- */ 36 + 37 + 38 + /** 39 + * @task update 40 + */ 25 41 private function updateLease(DrydockLease $lease) { 26 - if (!$lease->canUpdate()) { 42 + $this->processLeaseCommands($lease); 43 + 44 + $lease_status = $lease->getStatus(); 45 + switch ($lease_status) { 46 + case DrydockLeaseStatus::STATUS_PENDING: 47 + $this->executeAllocator($lease); 48 + break; 49 + case DrydockLeaseStatus::STATUS_ACQUIRED: 50 + $this->activateLease($lease); 51 + break; 52 + case DrydockLeaseStatus::STATUS_ACTIVE: 53 + // Nothing to do. 54 + break; 55 + case DrydockLeaseStatus::STATUS_RELEASED: 56 + case DrydockLeaseStatus::STATUS_BROKEN: 57 + $this->destroyLease($lease); 58 + break; 59 + case DrydockLeaseStatus::STATUS_DESTROYED: 60 + break; 61 + } 62 + 63 + $this->yieldIfExpiringLease($lease); 64 + } 65 + 66 + 67 + /* -( Processing Commands )------------------------------------------------ */ 68 + 69 + 70 + /** 71 + * @task command 72 + */ 73 + private function processLeaseCommands(DrydockLease $lease) { 74 + if (!$lease->canReceiveCommands()) { 27 75 return; 28 76 } 29 77 ··· 31 79 32 80 $commands = $this->loadCommands($lease->getPHID()); 33 81 foreach ($commands as $command) { 34 - if (!$lease->canUpdate()) { 82 + if (!$lease->canReceiveCommands()) { 35 83 break; 36 84 } 37 85 38 - $this->processCommand($lease, $command); 86 + $this->processLeaseCommand($lease, $command); 39 87 40 88 $command 41 89 ->setIsConsumed(true) 42 90 ->save(); 43 91 } 92 + } 44 93 45 - $this->yieldIfExpiringLease($lease); 46 - } 47 94 48 - private function processCommand( 95 + /** 96 + * @task command 97 + */ 98 + private function processLeaseCommand( 49 99 DrydockLease $lease, 50 100 DrydockCommand $command) { 51 101 switch ($command->getCommand()) { ··· 55 105 } 56 106 } 57 107 108 + 109 + /* -( Drydock Allocator )-------------------------------------------------- */ 110 + 111 + 112 + /** 113 + * Find or build a resource which can satisfy a given lease request, then 114 + * acquire the lease. 115 + * 116 + * @param DrydockLease Requested lease. 117 + * @return void 118 + * @task allocator 119 + */ 120 + private function executeAllocator(DrydockLease $lease) { 121 + $blueprints = $this->loadBlueprintsForAllocatingLease($lease); 122 + 123 + // If we get nothing back, that means no blueprint is defined which can 124 + // ever build the requested resource. This is a permanent failure, since 125 + // we don't expect to succeed no matter how many times we try. 126 + if (!$blueprints) { 127 + throw new PhabricatorWorkerPermanentFailureException( 128 + pht( 129 + 'No active Drydock blueprint exists which can ever allocate a '. 130 + 'resource for lease "%s".', 131 + $lease->getPHID())); 132 + } 133 + 134 + // First, try to find a suitable open resource which we can acquire a new 135 + // lease on. 136 + $resources = $this->loadResourcesForAllocatingLease($blueprints, $lease); 137 + 138 + // If no resources exist yet, see if we can build one. 139 + if (!$resources) { 140 + $usable_blueprints = $this->removeOverallocatedBlueprints( 141 + $blueprints, 142 + $lease); 143 + 144 + // If we get nothing back here, some blueprint claims it can eventually 145 + // satisfy the lease, just not right now. This is a temporary failure, 146 + // and we expect allocation to succeed eventually. 147 + if (!$usable_blueprints) { 148 + // TODO: More formal temporary failure here. We should retry this 149 + // "soon" but not "immediately". 150 + throw new Exception( 151 + pht('No blueprints have space to allocate a resource right now.')); 152 + } 153 + 154 + $usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease); 155 + 156 + $exceptions = array(); 157 + foreach ($usable_blueprints as $blueprint) { 158 + try { 159 + $resources[] = $this->allocateResource($blueprint, $lease); 160 + 161 + // Bail after allocating one resource, we don't need any more than 162 + // this. 163 + break; 164 + } catch (Exception $ex) { 165 + $exceptions[] = $ex; 166 + } 167 + } 168 + 169 + if (!$resources) { 170 + // TODO: We should distinguish between temporary and permament failures 171 + // here. If any blueprint failed temporarily, retry "soon". If none 172 + // of these failures were temporary, maybe this should be a permanent 173 + // failure? 174 + throw new PhutilAggregateException( 175 + pht( 176 + 'All blueprints failed to allocate a suitable new resource when '. 177 + 'trying to allocate lease "%s".', 178 + $lease->getPHID()), 179 + $exceptions); 180 + } 181 + 182 + // NOTE: We have not acquired the lease yet, so it is possible that the 183 + // resource we just built will be snatched up by some other lease before 184 + // we can. This is not problematic: we'll retry a little later and should 185 + // suceed eventually. 186 + } 187 + 188 + $resources = $this->rankResources($resources, $lease); 189 + 190 + $exceptions = array(); 191 + $allocated = false; 192 + foreach ($resources as $resource) { 193 + try { 194 + $this->acquireLease($resource, $lease); 195 + $allocated = true; 196 + break; 197 + } catch (Exception $ex) { 198 + $exceptions[] = $ex; 199 + } 200 + } 201 + 202 + if (!$allocated) { 203 + // TODO: We should distinguish between temporary and permanent failures 204 + // here. If any failures were temporary (specifically, failed to acquire 205 + // locks) 206 + 207 + throw new PhutilAggregateException( 208 + pht( 209 + 'Unable to acquire lease "%s" on any resouce.', 210 + $lease->getPHID()), 211 + $exceptions); 212 + } 213 + } 214 + 215 + 216 + /** 217 + * Get all the @{class:DrydockBlueprintImplementation}s which can possibly 218 + * build a resource to satisfy a lease. 219 + * 220 + * This method returns blueprints which might, at some time, be able to 221 + * build a resource which can satisfy the lease. They may not be able to 222 + * build that resource right now. 223 + * 224 + * @param DrydockLease Requested lease. 225 + * @return list<DrydockBlueprintImplementation> List of qualifying blueprint 226 + * implementations. 227 + * @task allocator 228 + */ 229 + private function loadBlueprintImplementationsForAllocatingLease( 230 + DrydockLease $lease) { 231 + 232 + $impls = DrydockBlueprintImplementation::getAllBlueprintImplementations(); 233 + 234 + $keep = array(); 235 + foreach ($impls as $key => $impl) { 236 + // Don't use disabled blueprint types. 237 + if (!$impl->isEnabled()) { 238 + continue; 239 + } 240 + 241 + // Don't use blueprint types which can't allocate the correct kind of 242 + // resource. 243 + if ($impl->getType() != $lease->getResourceType()) { 244 + continue; 245 + } 246 + 247 + if (!$impl->canAnyBlueprintEverAllocateResourceForLease($lease)) { 248 + continue; 249 + } 250 + 251 + $keep[$key] = $impl; 252 + } 253 + 254 + return $keep; 255 + } 256 + 257 + 258 + /** 259 + * Get all the concrete @{class:DrydockBlueprint}s which can possibly 260 + * build a resource to satisfy a lease. 261 + * 262 + * @param DrydockLease Requested lease. 263 + * @return list<DrydockBlueprint> List of qualifying blueprints. 264 + * @task allocator 265 + */ 266 + private function loadBlueprintsForAllocatingLease( 267 + DrydockLease $lease) { 268 + $viewer = $this->getViewer(); 269 + 270 + $impls = $this->loadBlueprintImplementationsForAllocatingLease($lease); 271 + if (!$impls) { 272 + return array(); 273 + } 274 + 275 + $blueprints = id(new DrydockBlueprintQuery()) 276 + ->setViewer($viewer) 277 + ->withBlueprintClasses(array_keys($impls)) 278 + ->withDisabled(false) 279 + ->execute(); 280 + 281 + $keep = array(); 282 + foreach ($blueprints as $key => $blueprint) { 283 + if (!$blueprint->canEverAllocateResourceForLease($lease)) { 284 + continue; 285 + } 286 + 287 + $keep[$key] = $blueprint; 288 + } 289 + 290 + return $keep; 291 + } 292 + 293 + 294 + /** 295 + * Load a list of all resources which a given lease can possibly be 296 + * allocated against. 297 + * 298 + * @param list<DrydockBlueprint> Blueprints which may produce suitable 299 + * resources. 300 + * @param DrydockLease Requested lease. 301 + * @return list<DrydockResource> Resources which may be able to allocate 302 + * the lease. 303 + * @task allocator 304 + */ 305 + private function loadResourcesForAllocatingLease( 306 + array $blueprints, 307 + DrydockLease $lease) { 308 + assert_instances_of($blueprints, 'DrydockBlueprint'); 309 + $viewer = $this->getViewer(); 310 + 311 + $resources = id(new DrydockResourceQuery()) 312 + ->setViewer($viewer) 313 + ->withBlueprintPHIDs(mpull($blueprints, 'getPHID')) 314 + ->withTypes(array($lease->getResourceType())) 315 + ->withStatuses( 316 + array( 317 + DrydockResourceStatus::STATUS_PENDING, 318 + DrydockResourceStatus::STATUS_ACTIVE, 319 + )) 320 + ->execute(); 321 + 322 + $keep = array(); 323 + foreach ($resources as $key => $resource) { 324 + $blueprint = $resource->getBlueprint(); 325 + 326 + if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) { 327 + continue; 328 + } 329 + 330 + $keep[$key] = $resource; 331 + } 332 + 333 + return $keep; 334 + } 335 + 336 + 337 + /** 338 + * Remove blueprints which are too heavily allocated to build a resource for 339 + * a lease from a list of blueprints. 340 + * 341 + * @param list<DrydockBlueprint> List of blueprints. 342 + * @return list<DrydockBlueprint> List with blueprints that can not allocate 343 + * a resource for the lease right now removed. 344 + * @task allocator 345 + */ 346 + private function removeOverallocatedBlueprints( 347 + array $blueprints, 348 + DrydockLease $lease) { 349 + assert_instances_of($blueprints, 'DrydockBlueprint'); 350 + 351 + $keep = array(); 352 + foreach ($blueprints as $key => $blueprint) { 353 + if (!$blueprint->canAllocateResourceForLease($lease)) { 354 + continue; 355 + } 356 + 357 + $keep[$key] = $blueprint; 358 + } 359 + 360 + return $keep; 361 + } 362 + 363 + 364 + /** 365 + * Rank blueprints by suitability for building a new resource for a 366 + * particular lease. 367 + * 368 + * @param list<DrydockBlueprint> List of blueprints. 369 + * @param DrydockLease Requested lease. 370 + * @return list<DrydockBlueprint> Ranked list of blueprints. 371 + * @task allocator 372 + */ 373 + private function rankBlueprints(array $blueprints, DrydockLease $lease) { 374 + assert_instances_of($blueprints, 'DrydockBlueprint'); 375 + 376 + // TODO: Implement improvements to this ranking algorithm if they become 377 + // available. 378 + shuffle($blueprints); 379 + 380 + return $blueprints; 381 + } 382 + 383 + 384 + /** 385 + * Rank resources by suitability for allocating a particular lease. 386 + * 387 + * @param list<DrydockResource> List of resources. 388 + * @param DrydockLease Requested lease. 389 + * @return list<DrydockResource> Ranked list of resources. 390 + * @task allocator 391 + */ 392 + private function rankResources(array $resources, DrydockLease $lease) { 393 + assert_instances_of($resources, 'DrydockResource'); 394 + 395 + // TODO: Implement improvements to this ranking algorithm if they become 396 + // available. 397 + shuffle($resources); 398 + 399 + return $resources; 400 + } 401 + 402 + 403 + /** 404 + * Perform an actual resource allocation with a particular blueprint. 405 + * 406 + * @param DrydockBlueprint The blueprint to allocate a resource from. 407 + * @param DrydockLease Requested lease. 408 + * @return DrydockResource Allocated resource. 409 + * @task allocator 410 + */ 411 + private function allocateResource( 412 + DrydockBlueprint $blueprint, 413 + DrydockLease $lease) { 414 + $resource = $blueprint->allocateResource($lease); 415 + $this->validateAllocatedResource($blueprint, $resource, $lease); 416 + 417 + // If this resource was allocated as a pending resource, queue a task to 418 + // activate it. 419 + if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) { 420 + PhabricatorWorker::scheduleTask( 421 + 'DrydockResourceUpdateWorker', 422 + array( 423 + 'resourcePHID' => $resource->getPHID(), 424 + ), 425 + array( 426 + 'objectPHID' => $resource->getPHID(), 427 + )); 428 + } 429 + 430 + return $resource; 431 + } 432 + 433 + 434 + /** 435 + * Check that the resource a blueprint allocated is roughly the sort of 436 + * object we expect. 437 + * 438 + * @param DrydockBlueprint Blueprint which built the resource. 439 + * @param wild Thing which the blueprint claims is a valid resource. 440 + * @param DrydockLease Lease the resource was allocated for. 441 + * @return void 442 + * @task allocator 443 + */ 444 + private function validateAllocatedResource( 445 + DrydockBlueprint $blueprint, 446 + $resource, 447 + DrydockLease $lease) { 448 + 449 + if (!($resource instanceof DrydockResource)) { 450 + throw new Exception( 451 + pht( 452 + 'Blueprint "%s" (of type "%s") is not properly implemented: %s must '. 453 + 'return an object of type %s or throw, but returned something else.', 454 + $blueprint->getBlueprintName(), 455 + $blueprint->getClassName(), 456 + 'allocateResource()', 457 + 'DrydockResource')); 458 + } 459 + 460 + if (!$resource->isAllocatedResource()) { 461 + throw new Exception( 462 + pht( 463 + 'Blueprint "%s" (of type "%s") is not properly implemented: %s '. 464 + 'must actually allocate the resource it returns.', 465 + $blueprint->getBlueprintName(), 466 + $blueprint->getClassName(), 467 + 'allocateResource()')); 468 + } 469 + 470 + $resource_type = $resource->getType(); 471 + $lease_type = $lease->getResourceType(); 472 + 473 + if ($resource_type !== $lease_type) { 474 + // TODO: Destroy the resource here? 475 + 476 + throw new Exception( 477 + pht( 478 + 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 479 + 'built a resource of type "%s" to satisfy a lease requesting a '. 480 + 'resource of type "%s".', 481 + $blueprint->getBlueprintName(), 482 + $blueprint->getClassName(), 483 + $resource_type, 484 + $lease_type)); 485 + } 486 + } 487 + 488 + 489 + /* -( Acquiring Leases )--------------------------------------------------- */ 490 + 491 + 492 + /** 493 + * Perform an actual lease acquisition on a particular resource. 494 + * 495 + * @param DrydockResource Resource to acquire a lease on. 496 + * @param DrydockLease Lease to acquire. 497 + * @return void 498 + * @task acquire 499 + */ 500 + private function acquireLease( 501 + DrydockResource $resource, 502 + DrydockLease $lease) { 503 + 504 + $blueprint = $resource->getBlueprint(); 505 + $blueprint->acquireLease($resource, $lease); 506 + 507 + $this->validateAcquiredLease($blueprint, $resource, $lease); 508 + 509 + // If this lease has been acquired but not activated, queue a task to 510 + // activate it. 511 + if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) { 512 + PhabricatorWorker::scheduleTask( 513 + __CLASS__, 514 + array( 515 + 'leasePHID' => $lease->getPHID(), 516 + ), 517 + array( 518 + 'objectPHID' => $lease->getPHID(), 519 + )); 520 + } 521 + } 522 + 523 + 524 + /** 525 + * Make sure that a lease was really acquired properly. 526 + * 527 + * @param DrydockBlueprint Blueprint which created the resource. 528 + * @param DrydockResource Resource which was acquired. 529 + * @param DrydockLease The lease which was supposedly acquired. 530 + * @return void 531 + * @task acquire 532 + */ 533 + private function validateAcquiredLease( 534 + DrydockBlueprint $blueprint, 535 + DrydockResource $resource, 536 + DrydockLease $lease) { 537 + 538 + if (!$lease->isAcquiredLease()) { 539 + throw new Exception( 540 + pht( 541 + 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 542 + 'returned from "%s" without acquiring a lease.', 543 + $blueprint->getBlueprintName(), 544 + $blueprint->getClassName(), 545 + 'acquireLease()')); 546 + } 547 + 548 + $lease_phid = $lease->getResourcePHID(); 549 + $resource_phid = $resource->getPHID(); 550 + 551 + if ($lease_phid !== $resource_phid) { 552 + // TODO: Destroy the lease? 553 + throw new Exception( 554 + pht( 555 + 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 556 + 'returned from "%s" with a lease acquired on the wrong resource.', 557 + $blueprint->getBlueprintName(), 558 + $blueprint->getClassName(), 559 + 'acquireLease()')); 560 + } 561 + } 562 + 563 + 564 + /* -( Activating Leases )-------------------------------------------------- */ 565 + 566 + 567 + /** 568 + * @task activate 569 + */ 570 + private function activateLease(DrydockLease $lease) { 571 + $resource = $lease->getResource(); 572 + if (!$resource) { 573 + throw new PhabricatorWorkerPermanentFailureException( 574 + pht('Trying to activate lease with no resource.')); 575 + } 576 + 577 + $resource_status = $resource->getStatus(); 578 + 579 + if ($resource_status == DrydockResourceStatus::STATUS_PENDING) { 580 + // TODO: This is explicitly a temporary failure -- we are waiting for 581 + // the resource to come up. 582 + throw new Exception(pht('Resource still activating.')); 583 + } 584 + 585 + if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) { 586 + throw new PhabricatorWorkerPermanentFailureException( 587 + pht( 588 + 'Trying to activate lease on a dead resource (in status "%s").', 589 + $resource_status)); 590 + } 591 + 592 + // NOTE: We can race resource destruction here. Between the time we 593 + // performed the read above and now, the resource might have closed, so 594 + // we may activate leases on dead resources. At least for now, this seems 595 + // fine: a resource dying right before we activate a lease on it should not 596 + // be distinguisahble from a resource dying right after we activate a lease 597 + // on it. We end up with an active lease on a dead resource either way, and 598 + // can not prevent resources dying from lightning strikes. 599 + 600 + $blueprint = $resource->getBlueprint(); 601 + $blueprint->activateLease($resource, $lease); 602 + $this->validateActivatedLease($blueprint, $resource, $lease); 603 + } 604 + 605 + /** 606 + * @task activate 607 + */ 608 + private function validateActivatedLease( 609 + DrydockBlueprint $blueprint, 610 + DrydockResource $resource, 611 + DrydockLease $lease) { 612 + 613 + if (!$lease->isActivatedLease()) { 614 + throw new Exception( 615 + pht( 616 + 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 617 + 'returned from "%s" without activating a lease.', 618 + $blueprint->getBlueprintName(), 619 + $blueprint->getClassName(), 620 + 'acquireLease()')); 621 + } 622 + 623 + } 624 + 625 + 626 + /* -( Releasing Leases )--------------------------------------------------- */ 627 + 628 + 629 + /** 630 + * @task release 631 + */ 58 632 private function releaseLease(DrydockLease $lease) { 59 633 $lease->openTransaction(); 60 634 $lease ··· 65 639 DrydockSlotLock::releaseLocks($lease->getPHID()); 66 640 $lease->saveTransaction(); 67 641 68 - PhabricatorWorker::scheduleTask( 69 - 'DrydockLeaseDestroyWorker', 70 - array( 71 - 'leasePHID' => $lease->getPHID(), 72 - ), 73 - array( 74 - 'objectPHID' => $lease->getPHID(), 75 - )); 76 - 77 642 $lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST); 78 643 79 644 $resource = $lease->getResource(); 80 645 $blueprint = $resource->getBlueprint(); 81 646 82 647 $blueprint->didReleaseLease($resource, $lease); 648 + 649 + $this->destroyLease($lease); 650 + } 651 + 652 + 653 + /* -( Destroying Leases )-------------------------------------------------- */ 654 + 655 + 656 + /** 657 + * @task destroy 658 + */ 659 + private function destroyLease(DrydockLease $lease) { 660 + $resource = $lease->getResource(); 661 + $blueprint = $resource->getBlueprint(); 662 + 663 + $blueprint->destroyLease($resource, $lease); 664 + 665 + $lease 666 + ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED) 667 + ->save(); 668 + 669 + $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST); 83 670 } 84 671 85 672 }

-74

src/applications/drydock/worker/DrydockLeaseWorker.php

··· 1 - <?php 2 - 3 - final class DrydockLeaseWorker extends DrydockWorker { 4 - 5 - protected function doWork() { 6 - $lease_phid = $this->getTaskDataValue('leasePHID'); 7 - $lease = $this->loadLease($lease_phid); 8 - 9 - $this->activateLease($lease); 10 - } 11 - 12 - 13 - private function activateLease(DrydockLease $lease) { 14 - $actual_status = $lease->getStatus(); 15 - 16 - if ($actual_status != DrydockLeaseStatus::STATUS_ACQUIRED) { 17 - throw new PhabricatorWorkerPermanentFailureException( 18 - pht( 19 - 'Trying to activate lease from wrong status ("%s").', 20 - $actual_status)); 21 - } 22 - 23 - $resource = $lease->getResource(); 24 - if (!$resource) { 25 - throw new PhabricatorWorkerPermanentFailureException( 26 - pht('Trying to activate lease with no resource.')); 27 - } 28 - 29 - $resource_status = $resource->getStatus(); 30 - 31 - if ($resource_status == DrydockResourceStatus::STATUS_PENDING) { 32 - // TODO: This is explicitly a temporary failure -- we are waiting for 33 - // the resource to come up. 34 - throw new Exception(pht('Resource still activating.')); 35 - } 36 - 37 - if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) { 38 - throw new PhabricatorWorkerPermanentFailureException( 39 - pht( 40 - 'Trying to activate lease on a dead resource (in status "%s").', 41 - $resource_status)); 42 - } 43 - 44 - // NOTE: We can race resource destruction here. Between the time we 45 - // performed the read above and now, the resource might have closed, so 46 - // we may activate leases on dead resources. At least for now, this seems 47 - // fine: a resource dying right before we activate a lease on it should not 48 - // be distinguisahble from a resource dying right after we activate a lease 49 - // on it. We end up with an active lease on a dead resource either way, and 50 - // can not prevent resources dying from lightning strikes. 51 - 52 - $blueprint = $resource->getBlueprint(); 53 - $blueprint->activateLease($resource, $lease); 54 - $this->validateActivatedLease($blueprint, $resource, $lease); 55 - } 56 - 57 - private function validateActivatedLease( 58 - DrydockBlueprint $blueprint, 59 - DrydockResource $resource, 60 - DrydockLease $lease) { 61 - 62 - if (!$lease->isActivatedLease()) { 63 - throw new Exception( 64 - pht( 65 - 'Blueprint "%s" (of type "%s") is not properly implemented: it '. 66 - 'returned from "%s" without activating a lease.', 67 - $blueprint->getBlueprintName(), 68 - $blueprint->getClassName(), 69 - 'acquireLease()')); 70 - } 71 - 72 - } 73 - 74 - }

+1 -1

src/applications/drydock/worker/DrydockWorker.php

··· 86 86 } 87 87 88 88 protected function yieldIfExpiringLease(DrydockLease $lease) { 89 - if (!$lease->canUpdate()) { 89 + if (!$lease->canReceiveCommands()) { 90 90 return; 91 91 } 92 92

+2 -1

src/applications/harbormaster/storage/build/HarbormasterBuildLog.php

··· 154 154 155 155 public function finalize($start = 0) { 156 156 if (!$this->getLive()) { 157 - throw new Exception(pht('Start logging before finalizing it.')); 157 + // TODO: Clean up this API. 158 + return; 158 159 } 159 160 160 161 // TODO: Encode the log contents in a gzipped format.

+4 -1

src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php

··· 60 60 $id = $this->getID(); 61 61 $class = $this->getTaskClass(); 62 62 63 - if (!class_exists($class)) { 63 + try { 64 + // NOTE: If the class does not exist, libphutil will throw an exception. 65 + class_exists($class); 66 + } catch (PhutilMissingSymbolException $ex) { 64 67 throw new PhabricatorWorkerPermanentFailureException( 65 68 pht( 66 69 "Task class '%s' does not exist!",

Configure Feed

Configure Feed