@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
1<?php
2
3/**
4 * @task update Updating Leases
5 * @task command Processing Commands
6 * @task allocator Drydock Allocator
7 * @task acquire Acquiring Leases
8 * @task activate Activating Leases
9 * @task release Releasing Leases
10 * @task break Breaking Leases
11 * @task destroy Destroying Leases
12 */
13final class DrydockLeaseUpdateWorker extends DrydockWorker {
14
15 protected function doWork() {
16 $lease_phid = $this->getTaskDataValue('leasePHID');
17
18 $hash = PhabricatorHash::digestForIndex($lease_phid);
19 $lock_key = 'drydock.lease:'.$hash;
20
21 $lock = PhabricatorGlobalLock::newLock($lock_key)
22 ->lock(1);
23
24 try {
25 $lease = $this->loadLease($lease_phid);
26 $this->handleUpdate($lease);
27 } catch (Exception $ex) {
28 $lock->unlock();
29 $this->flushDrydockTaskQueue();
30 throw $ex;
31 }
32
33 $lock->unlock();
34 }
35
36
37/* -( Updating Leases )---------------------------------------------------- */
38
39
40 /**
41 * @task update
42 */
43 private function handleUpdate(DrydockLease $lease) {
44 try {
45 $this->updateLease($lease);
46 } catch (DrydockAcquiredBrokenResourceException $ex) {
47 // If this lease acquired a resource but failed to activate, we don't
48 // need to break the lease. We can throw it back in the pool and let
49 // it take another shot at acquiring a new resource.
50
51 // Before we throw it back, release any locks the lease is holding.
52 DrydockSlotLock::releaseLocks($lease->getPHID());
53
54 $lease
55 ->setStatus(DrydockLeaseStatus::STATUS_PENDING)
56 ->setResourcePHID(null)
57 ->save();
58
59 $lease->logEvent(
60 DrydockLeaseReacquireLogType::LOGCONST,
61 array(
62 'class' => get_class($ex),
63 'message' => $ex->getMessage(),
64 ));
65
66 $this->yieldLease($lease, $ex);
67 } catch (Exception $ex) {
68 if ($this->isTemporaryException($ex)) {
69 $this->yieldLease($lease, $ex);
70 } else {
71 $this->breakLease($lease, $ex);
72 }
73 }
74 }
75
76
77 /**
78 * @task update
79 */
80 private function updateLease(DrydockLease $lease) {
81 $this->processLeaseCommands($lease);
82
83 $lease_status = $lease->getStatus();
84 switch ($lease_status) {
85 case DrydockLeaseStatus::STATUS_PENDING:
86 $this->executeAllocator($lease);
87 break;
88 case DrydockLeaseStatus::STATUS_ACQUIRED:
89 $this->activateLease($lease);
90 break;
91 case DrydockLeaseStatus::STATUS_ACTIVE:
92 // Nothing to do.
93 break;
94 case DrydockLeaseStatus::STATUS_RELEASED:
95 case DrydockLeaseStatus::STATUS_BROKEN:
96 $this->destroyLease($lease);
97 break;
98 case DrydockLeaseStatus::STATUS_DESTROYED:
99 break;
100 }
101
102 $this->yieldIfExpiringLease($lease);
103 }
104
105
106 /**
107 * @task update
108 */
109 private function yieldLease(DrydockLease $lease, Exception $ex) {
110 $duration = $this->getYieldDurationFromException($ex);
111
112 $lease->logEvent(
113 DrydockLeaseActivationYieldLogType::LOGCONST,
114 array(
115 'duration' => $duration,
116 ));
117
118 throw new PhabricatorWorkerYieldException($duration);
119 }
120
121
122/* -( Processing Commands )------------------------------------------------ */
123
124
125 /**
126 * @task command
127 */
128 private function processLeaseCommands(DrydockLease $lease) {
129 if (!$lease->canReceiveCommands()) {
130 return;
131 }
132
133 $this->checkLeaseExpiration($lease);
134
135 $commands = $this->loadCommands($lease->getPHID());
136 foreach ($commands as $command) {
137 if (!$lease->canReceiveCommands()) {
138 break;
139 }
140
141 $this->processLeaseCommand($lease, $command);
142
143 $command
144 ->setIsConsumed(true)
145 ->save();
146 }
147 }
148
149
150 /**
151 * @task command
152 */
153 private function processLeaseCommand(
154 DrydockLease $lease,
155 DrydockCommand $command) {
156 switch ($command->getCommand()) {
157 case DrydockCommand::COMMAND_RELEASE:
158 $this->releaseLease($lease);
159 break;
160 }
161 }
162
163
164/* -( Drydock Allocator )-------------------------------------------------- */
165
166
167 /**
168 * Find or build a resource which can satisfy a given lease request, then
169 * acquire the lease.
170 *
171 * @param DrydockLease $lease Requested lease.
172 * @return DrydockResource
173 * @task allocator
174 */
175 private function executeAllocator(DrydockLease $lease) {
176 $blueprints = $this->loadBlueprintsForAllocatingLease($lease);
177
178 // If we get nothing back, that means no blueprint is defined which can
179 // ever build the requested resource. This is a permanent failure, since
180 // we don't expect to succeed no matter how many times we try.
181 if (!$blueprints) {
182 throw new PhabricatorWorkerPermanentFailureException(
183 pht(
184 'No active Drydock blueprint exists which can ever allocate a '.
185 'resource for lease "%s".',
186 $lease->getPHID()));
187 }
188
189 // First, try to find a suitable open resource which we can acquire a new
190 // lease on.
191 $resources = $this->loadAcquirableResourcesForLease($blueprints, $lease);
192
193 list($free_resources, $used_resources) = $this->partitionResources(
194 $lease,
195 $resources);
196
197 $resource = $this->leaseAnyResource($lease, $free_resources);
198 if ($resource) {
199 return $resource;
200 }
201
202 // We're about to try creating a resource. If we're already creating
203 // something, just yield until that resolves.
204
205 $this->yieldForPendingResources($lease);
206
207 // We haven't been able to lease an existing resource yet, so now we try to
208 // create one. We may still have some less-desirable "used" resources that
209 // we'll sometimes try to lease later if we fail to allocate a new resource.
210
211 $resource = $this->newLeasedResource($lease, $blueprints);
212 if ($resource) {
213 return $resource;
214 }
215
216 // We haven't been able to lease a desirable "free" resource or create a
217 // new resource. Try to lease a "used" resource.
218
219 $resource = $this->leaseAnyResource($lease, $used_resources);
220 if ($resource) {
221 return $resource;
222 }
223
224 // If this lease has already triggered a reclaim, just yield and wait for
225 // it to resolve.
226 $this->yieldForReclaimingResources($lease);
227
228 // Try to reclaim a resource. This will yield if it reclaims something.
229 $this->reclaimAnyResource($lease, $blueprints);
230
231 // We weren't able to lease, create, or reclaim any resources. We just have
232 // to wait for resources to become available.
233
234 $lease->logEvent(
235 DrydockLeaseWaitingForResourcesLogType::LOGCONST,
236 array(
237 'blueprintPHIDs' => mpull($blueprints, 'getPHID'),
238 ));
239
240 throw new PhabricatorWorkerYieldException(15);
241 }
242
243 /**
244 * @param DrydockLease $lease
245 * @param array<DrydockBlueprint> $blueprints
246 */
247 private function reclaimAnyResource(DrydockLease $lease, array $blueprints) {
248 assert_instances_of($blueprints, DrydockBlueprint::class);
249
250 $blueprints = $this->rankBlueprints($blueprints, $lease);
251
252 // Try to actively reclaim unused resources. If we succeed, jump back
253 // into the queue in an effort to claim it.
254
255 foreach ($blueprints as $blueprint) {
256 $reclaimed = $this->reclaimResources($blueprint, $lease);
257 if ($reclaimed) {
258
259 $lease->logEvent(
260 DrydockLeaseReclaimLogType::LOGCONST,
261 array(
262 'resourcePHIDs' => array($reclaimed->getPHID()),
263 ));
264
265 // Yield explicitly here: we'll be awakened when the resource is
266 // reclaimed.
267
268 throw new PhabricatorWorkerYieldException(15);
269 }
270 }
271 }
272
273 private function yieldForPendingResources(DrydockLease $lease) {
274 // See T13677. If this lease has already triggered the allocation of
275 // one or more resources and they are still pending, just yield and
276 // wait for them.
277
278 $viewer = $this->getViewer();
279
280 $phids = $lease->getAllocatedResourcePHIDs();
281 if (!$phids) {
282 return;
283 }
284
285 $resources = id(new DrydockResourceQuery())
286 ->setViewer($viewer)
287 ->withPHIDs($phids)
288 ->withStatuses(
289 array(
290 DrydockResourceStatus::STATUS_PENDING,
291 ))
292 ->setLimit(1)
293 ->execute();
294 if (!$resources) {
295 return;
296 }
297
298 $lease->logEvent(
299 DrydockLeaseWaitingForActivationLogType::LOGCONST,
300 array(
301 'resourcePHIDs' => mpull($resources, 'getPHID'),
302 ));
303
304 throw new PhabricatorWorkerYieldException(15);
305 }
306
307 private function yieldForReclaimingResources(DrydockLease $lease) {
308 $viewer = $this->getViewer();
309
310 $phids = $lease->getReclaimedResourcePHIDs();
311 if (!$phids) {
312 return;
313 }
314
315 $resources = id(new DrydockResourceQuery())
316 ->setViewer($viewer)
317 ->withPHIDs($phids)
318 ->withStatuses(
319 array(
320 DrydockResourceStatus::STATUS_ACTIVE,
321 DrydockResourceStatus::STATUS_RELEASED,
322 ))
323 ->setLimit(1)
324 ->execute();
325 if (!$resources) {
326 return;
327 }
328
329 $lease->logEvent(
330 DrydockLeaseWaitingForReclamationLogType::LOGCONST,
331 array(
332 'resourcePHIDs' => mpull($resources, 'getPHID'),
333 ));
334
335 throw new PhabricatorWorkerYieldException(15);
336 }
337
338 /**
339 * @param DrydockLease $lease
340 * @param array<DrydockBlueprint> $blueprints
341 */
342 private function newLeasedResource(
343 DrydockLease $lease,
344 array $blueprints) {
345 assert_instances_of($blueprints, DrydockBlueprint::class);
346
347 $usable_blueprints = $this->removeOverallocatedBlueprints(
348 $blueprints,
349 $lease);
350
351 // If we get nothing back here, some blueprint claims it can eventually
352 // satisfy the lease, just not right now. This is a temporary failure,
353 // and we expect allocation to succeed eventually.
354
355 // Return, try to lease a "used" resource, and continue from there.
356
357 if (!$usable_blueprints) {
358 return null;
359 }
360
361 $usable_blueprints = $this->rankBlueprints($usable_blueprints, $lease);
362
363 $new_resources = $this->newResources($lease, $usable_blueprints);
364 if (!$new_resources) {
365 // If we were unable to create any new resources, return and
366 // try to lease a "used" resource.
367 return null;
368 }
369
370 $new_resources = $this->removeUnacquirableResources(
371 $new_resources,
372 $lease);
373 if (!$new_resources) {
374 // If we make it here, we just built a resource but aren't allowed
375 // to acquire it. We expect this to happen if the resource prevents
376 // acquisition until it activates, which is common when a resource
377 // needs to perform setup steps.
378
379 // Explicitly yield and wait for activation, since we don't want to
380 // lease a "used" resource.
381
382 throw new PhabricatorWorkerYieldException(15);
383 }
384
385 $resource = $this->leaseAnyResource($lease, $new_resources);
386 if ($resource) {
387 return $resource;
388 }
389
390 // We may not be able to lease a resource even if we just built it:
391 // another process may snatch it up before we can lease it. This should
392 // be rare, but is not concerning. Just try to build another resource.
393
394 // We likely could try to build the next resource immediately, but err on
395 // the side of caution and yield for now, at least until this code is
396 // better vetted.
397
398 throw new PhabricatorWorkerYieldException(15);
399 }
400
401 /**
402 * @param DrydockLease $lease
403 * @param array<DrydockResource> $resources
404 */
405 private function partitionResources(
406 DrydockLease $lease,
407 array $resources) {
408
409 assert_instances_of($resources, DrydockResource::class);
410 $viewer = $this->getViewer();
411
412 $lease_statuses = array(
413 DrydockLeaseStatus::STATUS_PENDING,
414 DrydockLeaseStatus::STATUS_ACQUIRED,
415 DrydockLeaseStatus::STATUS_ACTIVE,
416 );
417
418 // Partition resources into "free" resources (which we can try to lease
419 // immediately) and "used" resources, which we can only to lease after we
420 // fail to allocate a new resource.
421
422 // "Free" resources are unleased and/or prefer reuse over allocation.
423 // "Used" resources are leased and prefer allocation over reuse.
424
425 $free_resources = array();
426 $used_resources = array();
427
428 foreach ($resources as $resource) {
429 $blueprint = $resource->getBlueprint();
430
431 if (!$blueprint->shouldAllocateSupplementalResource($resource, $lease)) {
432 $free_resources[] = $resource;
433 continue;
434 }
435
436 $leases = id(new DrydockLeaseQuery())
437 ->setViewer($viewer)
438 ->withResourcePHIDs(array($resource->getPHID()))
439 ->withStatuses($lease_statuses)
440 ->setLimit(1)
441 ->execute();
442 if (!$leases) {
443 $free_resources[] = $resource;
444 continue;
445 }
446
447 $used_resources[] = $resource;
448 }
449
450 return array($free_resources, $used_resources);
451 }
452
453 /**
454 * @param DrydockLease $lease
455 * @param array<DrydockBlueprint> $blueprints
456 */
457 private function newResources(
458 DrydockLease $lease,
459 array $blueprints) {
460 assert_instances_of($blueprints, DrydockBlueprint::class);
461
462 $resources = array();
463 $exceptions = array();
464 foreach ($blueprints as $blueprint) {
465 $caught = null;
466 try {
467 $resources[] = $this->allocateResource($blueprint, $lease);
468
469 // Bail after allocating one resource, we don't need any more than
470 // this.
471 break;
472 } catch (Exception $ex) {
473 $caught = $ex;
474 } catch (Throwable $ex) {
475 $caught = $ex;
476 }
477
478 if ($caught) {
479 // This failure is not normally expected, so log it. It can be
480 // caused by something mundane and recoverable, however (see below
481 // for discussion).
482
483 // We log to the blueprint separately from the log to the lease:
484 // the lease is not attached to a blueprint yet so the lease log
485 // will not show up on the blueprint; more than one blueprint may
486 // fail; and the lease is not really impacted (and won't log) if at
487 // least one blueprint actually works.
488
489 $blueprint->logEvent(
490 DrydockResourceAllocationFailureLogType::LOGCONST,
491 array(
492 'class' => get_class($caught),
493 'message' => $caught->getMessage(),
494 ));
495
496 $exceptions[] = $caught;
497 }
498 }
499
500 if (!$resources) {
501 // If one or more blueprints claimed that they would be able to allocate
502 // resources but none are actually able to allocate resources, log the
503 // failure and yield so we try again soon.
504
505 // This can happen if some unexpected issue occurs during allocation
506 // (for example, a call to build a VM fails for some reason) or if we
507 // raced another allocator and the blueprint is now full.
508
509 $ex = new PhutilAggregateException(
510 pht(
511 'All blueprints failed to allocate a suitable new resource when '.
512 'trying to allocate lease ("%s").',
513 $lease->getPHID()),
514 $exceptions);
515
516 $lease->logEvent(
517 DrydockLeaseAllocationFailureLogType::LOGCONST,
518 array(
519 'class' => get_class($ex),
520 'message' => $ex->getMessage(),
521 ));
522
523 return null;
524 }
525
526 return $resources;
527 }
528
529 /**
530 * @param DrydockLease $lease
531 * @param array<DrydockResource> $resources
532 */
533 private function leaseAnyResource(
534 DrydockLease $lease,
535 array $resources) {
536 assert_instances_of($resources, DrydockResource::class);
537
538 if (!$resources) {
539 return null;
540 }
541
542 $resources = $this->rankResources($resources, $lease);
543
544 $exceptions = array();
545 $yields = array();
546
547 $allocated = null;
548 foreach ($resources as $resource) {
549 try {
550 $this->acquireLease($resource, $lease);
551 $allocated = $resource;
552 break;
553 } catch (DrydockResourceLockException $ex) {
554 // We need to lock the resource to actually acquire it. If we aren't
555 // able to acquire the lock quickly enough, we can yield and try again
556 // later.
557 $yields[] = $ex;
558 } catch (DrydockSlotLockException $ex) {
559 // This also just indicates we ran into some kind of contention,
560 // probably from another lease. Just yield.
561 $yields[] = $ex;
562 } catch (DrydockAcquiredBrokenResourceException $ex) {
563 // If a resource was reclaimed or destroyed by the time we actually
564 // got around to acquiring it, we just got unlucky.
565 $yields[] = $ex;
566 } catch (PhabricatorWorkerYieldException $ex) {
567 // We can be told to yield, particularly by the supplemental allocator
568 // trying to give us a supplemental resource.
569 $yields[] = $ex;
570 } catch (Exception $ex) {
571 $exceptions[] = $ex;
572 }
573 }
574
575 if ($allocated) {
576 return $allocated;
577 }
578
579 if ($yields) {
580 throw new PhabricatorWorkerYieldException(15);
581 }
582
583 throw new PhutilAggregateException(
584 pht(
585 'Unable to acquire lease "%s" on any resource.',
586 $lease->getPHID()),
587 $exceptions);
588 }
589
590
591 /**
592 * Get all the concrete @{class:DrydockBlueprint}s which can possibly
593 * build a resource to satisfy a lease.
594 *
595 * @param DrydockLease $lease Requested lease.
596 * @return list<DrydockBlueprint> List of qualifying blueprints.
597 * @task allocator
598 */
599 private function loadBlueprintsForAllocatingLease(
600 DrydockLease $lease) {
601 $viewer = $this->getViewer();
602
603 $impls = DrydockBlueprintImplementation::getAllForAllocatingLease($lease);
604 if (!$impls) {
605 return array();
606 }
607
608 $blueprint_phids = $lease->getAllowedBlueprintPHIDs();
609 if (!$blueprint_phids) {
610 $lease->logEvent(DrydockLeaseNoBlueprintsLogType::LOGCONST);
611 return array();
612 }
613
614 $query = id(new DrydockBlueprintQuery())
615 ->setViewer($viewer)
616 ->withPHIDs($blueprint_phids)
617 ->withBlueprintClasses(array_keys($impls))
618 ->withDisabled(false);
619
620 // The Drydock application itself is allowed to authorize anything. This
621 // is primarily used for leases generated by CLI administrative tools.
622 $drydock_phid = id(new PhabricatorDrydockApplication())->getPHID();
623
624 $authorizing_phid = $lease->getAuthorizingPHID();
625 if ($authorizing_phid != $drydock_phid) {
626 $blueprints = id(clone $query)
627 ->withAuthorizedPHIDs(array($authorizing_phid))
628 ->execute();
629 if (!$blueprints) {
630 // If we didn't hit any blueprints, check if this is an authorization
631 // problem: re-execute the query without the authorization constraint.
632 // If the second query hits blueprints, the overall configuration is
633 // fine but this is an authorization problem. If the second query also
634 // comes up blank, this is some other kind of configuration issue so
635 // we fall through to the default pathway.
636 $all_blueprints = $query->execute();
637 if ($all_blueprints) {
638 $lease->logEvent(
639 DrydockLeaseNoAuthorizationsLogType::LOGCONST,
640 array(
641 'authorizingPHID' => $authorizing_phid,
642 ));
643 return array();
644 }
645 }
646 } else {
647 $blueprints = $query->execute();
648 }
649
650 $keep = array();
651 foreach ($blueprints as $key => $blueprint) {
652 if (!$blueprint->canEverAllocateResourceForLease($lease)) {
653 continue;
654 }
655
656 $keep[$key] = $blueprint;
657 }
658
659 return $keep;
660 }
661
662
663 /**
664 * Load a list of all resources which a given lease can possibly be
665 * allocated against.
666 *
667 * @param array<DrydockBlueprint> $blueprints Blueprints which may produce
668 * suitable resources.
669 * @param DrydockLease $lease Requested lease.
670 * @return array<DrydockResource> Resources which may be able to allocate
671 * the lease.
672 * @task allocator
673 */
674 private function loadAcquirableResourcesForLease(
675 array $blueprints,
676 DrydockLease $lease) {
677 assert_instances_of($blueprints, DrydockBlueprint::class);
678 $viewer = $this->getViewer();
679
680 $resources = id(new DrydockResourceQuery())
681 ->setViewer($viewer)
682 ->withBlueprintPHIDs(mpull($blueprints, 'getPHID'))
683 ->withTypes(array($lease->getResourceType()))
684 ->withStatuses(
685 array(
686 DrydockResourceStatus::STATUS_ACTIVE,
687 ))
688 ->execute();
689
690 return $this->removeUnacquirableResources($resources, $lease);
691 }
692
693
694 /**
695 * Remove resources which can not be acquired by a given lease from a list.
696 *
697 * @param list<DrydockResource> $resources Candidate resources.
698 * @param DrydockLease $lease Acquiring lease.
699 * @return list<DrydockResource> Resources which the lease may be able to
700 * acquire.
701 * @task allocator
702 */
703 private function removeUnacquirableResources(
704 array $resources,
705 DrydockLease $lease) {
706 $keep = array();
707 foreach ($resources as $key => $resource) {
708 $blueprint = $resource->getBlueprint();
709
710 if (!$blueprint->canAcquireLeaseOnResource($resource, $lease)) {
711 continue;
712 }
713
714 $keep[$key] = $resource;
715 }
716
717 return $keep;
718 }
719
720
721 /**
722 * Remove blueprints which are too heavily allocated to build a resource for
723 * a lease from a list of blueprints.
724 *
725 * @param array<DrydockBlueprint> $blueprints List of blueprints.
726 * @param DrydockLease $lease
727 * @return array<DrydockBlueprint> $lease List with blueprints that can not
728 * allocate a resource for the lease right now removed.
729 * @task allocator
730 */
731 private function removeOverallocatedBlueprints(
732 array $blueprints,
733 DrydockLease $lease) {
734 assert_instances_of($blueprints, DrydockBlueprint::class);
735
736 $keep = array();
737
738 foreach ($blueprints as $key => $blueprint) {
739 if (!$blueprint->canAllocateResourceForLease($lease)) {
740 continue;
741 }
742
743 $keep[$key] = $blueprint;
744 }
745
746 return $keep;
747 }
748
749
750 /**
751 * Rank blueprints by suitability for building a new resource for a
752 * particular lease.
753 *
754 * @param array<DrydockBlueprint> $blueprints List of blueprints.
755 * @param DrydockLease $lease Requested lease.
756 * @return array<DrydockBlueprint> Ranked list of blueprints.
757 * @task allocator
758 */
759 private function rankBlueprints(array $blueprints, DrydockLease $lease) {
760 assert_instances_of($blueprints, DrydockBlueprint::class);
761
762 // TODO: Implement improvements to this ranking algorithm if they become
763 // available.
764 shuffle($blueprints);
765
766 return $blueprints;
767 }
768
769
770 /**
771 * Rank resources by suitability for allocating a particular lease.
772 *
773 * @param array<DrydockResource> $resources List of resources.
774 * @param DrydockLease $lease Requested lease.
775 * @return array<DrydockResource> Ranked list of resources.
776 * @task allocator
777 */
778 private function rankResources(array $resources, DrydockLease $lease) {
779 assert_instances_of($resources, DrydockResource::class);
780
781 // TODO: Implement improvements to this ranking algorithm if they become
782 // available.
783 shuffle($resources);
784
785 return $resources;
786 }
787
788
789 /**
790 * Perform an actual resource allocation with a particular blueprint.
791 *
792 * @param DrydockBlueprint $blueprint The blueprint to allocate a resource
793 * from.
794 * @param DrydockLease $lease Requested lease.
795 * @return DrydockResource Allocated resource.
796 * @task allocator
797 */
798 private function allocateResource(
799 DrydockBlueprint $blueprint,
800 DrydockLease $lease) {
801 $resource = $blueprint->allocateResource($lease);
802 $this->validateAllocatedResource($blueprint, $resource, $lease);
803
804 // If this resource was allocated as a pending resource, queue a task to
805 // activate it.
806 if ($resource->getStatus() == DrydockResourceStatus::STATUS_PENDING) {
807
808 $lease->addAllocatedResourcePHIDs(
809 array(
810 $resource->getPHID(),
811 ));
812 $lease->save();
813
814 PhabricatorWorker::scheduleTask(
815 'DrydockResourceUpdateWorker',
816 array(
817 'resourcePHID' => $resource->getPHID(),
818
819 // This task will generally yield while the resource activates, so
820 // wake it back up once the resource comes online. Most of the time,
821 // we'll be able to lease the newly activated resource.
822 'awakenOnActivation' => array(
823 $this->getCurrentWorkerTaskID(),
824 ),
825 ),
826 array(
827 'objectPHID' => $resource->getPHID(),
828 ));
829 }
830
831 return $resource;
832 }
833
834
835 /**
836 * Check that the resource a blueprint allocated is roughly the sort of
837 * object we expect.
838 *
839 * @param DrydockBlueprint $blueprint Blueprint which built the resource.
840 * @param mixed $resource Thing which the blueprint claims is a valid
841 * resource.
842 * @param DrydockLease $lease Lease the resource was allocated for.
843 * @return void
844 * @task allocator
845 */
846 private function validateAllocatedResource(
847 DrydockBlueprint $blueprint,
848 $resource,
849 DrydockLease $lease) {
850
851 if (!($resource instanceof DrydockResource)) {
852 throw new Exception(
853 pht(
854 'Blueprint "%s" (of type "%s") is not properly implemented: %s must '.
855 'return an object of type %s or throw, but returned something else.',
856 $blueprint->getBlueprintName(),
857 $blueprint->getClassName(),
858 'allocateResource()',
859 'DrydockResource'));
860 }
861
862 if (!$resource->isAllocatedResource()) {
863 throw new Exception(
864 pht(
865 'Blueprint "%s" (of type "%s") is not properly implemented: %s '.
866 'must actually allocate the resource it returns.',
867 $blueprint->getBlueprintName(),
868 $blueprint->getClassName(),
869 'allocateResource()'));
870 }
871
872 $resource_type = $resource->getType();
873 $lease_type = $lease->getResourceType();
874
875 if ($resource_type !== $lease_type) {
876 throw new Exception(
877 pht(
878 'Blueprint "%s" (of type "%s") is not properly implemented: it '.
879 'built a resource of type "%s" to satisfy a lease requesting a '.
880 'resource of type "%s".',
881 $blueprint->getBlueprintName(),
882 $blueprint->getClassName(),
883 $resource_type,
884 $lease_type));
885 }
886 }
887
888 private function reclaimResources(
889 DrydockBlueprint $blueprint,
890 DrydockLease $lease) {
891 $viewer = $this->getViewer();
892
893 $resources = id(new DrydockResourceQuery())
894 ->setViewer($viewer)
895 ->withBlueprintPHIDs(array($blueprint->getPHID()))
896 ->withStatuses(
897 array(
898 DrydockResourceStatus::STATUS_ACTIVE,
899 ))
900 ->execute();
901
902 // TODO: We could be much smarter about this and try to release long-unused
903 // resources, resources with many similar copies, old resources, resources
904 // that are cheap to rebuild, etc.
905 shuffle($resources);
906
907 foreach ($resources as $resource) {
908 if ($this->canReclaimResource($resource)) {
909 $this->reclaimResource($resource, $lease);
910 return $resource;
911 }
912 }
913
914 return null;
915 }
916
917
918/* -( Acquiring Leases )--------------------------------------------------- */
919
920
921 /**
922 * Perform an actual lease acquisition on a particular resource.
923 *
924 * @param DrydockResource $resource Resource to acquire a lease on.
925 * @param DrydockLease $lease Lease to acquire.
926 * @return void
927 * @task acquire
928 */
929 private function acquireLease(
930 DrydockResource $resource,
931 DrydockLease $lease) {
932
933 $blueprint = $resource->getBlueprint();
934 $blueprint->acquireLease($resource, $lease);
935
936 $this->validateAcquiredLease($blueprint, $resource, $lease);
937
938 // If this lease has been acquired but not activated, queue a task to
939 // activate it.
940 if ($lease->getStatus() == DrydockLeaseStatus::STATUS_ACQUIRED) {
941 $this->queueTask(
942 self::class,
943 array(
944 'leasePHID' => $lease->getPHID(),
945 ),
946 array(
947 'objectPHID' => $lease->getPHID(),
948 ));
949 }
950 }
951
952
953 /**
954 * Make sure that a lease was really acquired properly.
955 *
956 * @param DrydockBlueprint $blueprint Blueprint which created the resource.
957 * @param DrydockResource $resource Resource which was acquired.
958 * @param DrydockLease $lease The lease which was supposedly acquired.
959 * @return void
960 * @task acquire
961 */
962 private function validateAcquiredLease(
963 DrydockBlueprint $blueprint,
964 DrydockResource $resource,
965 DrydockLease $lease) {
966
967 if (!$lease->isAcquiredLease()) {
968 throw new Exception(
969 pht(
970 'Blueprint "%s" (of type "%s") is not properly implemented: it '.
971 'returned from "%s" without acquiring a lease.',
972 $blueprint->getBlueprintName(),
973 $blueprint->getClassName(),
974 'acquireLease()'));
975 }
976
977 $lease_phid = $lease->getResourcePHID();
978 $resource_phid = $resource->getPHID();
979
980 if ($lease_phid !== $resource_phid) {
981 throw new Exception(
982 pht(
983 'Blueprint "%s" (of type "%s") is not properly implemented: it '.
984 'returned from "%s" with a lease acquired on the wrong resource.',
985 $blueprint->getBlueprintName(),
986 $blueprint->getClassName(),
987 'acquireLease()'));
988 }
989 }
990
991
992/* -( Activating Leases )-------------------------------------------------- */
993
994
995 /**
996 * @task activate
997 */
998 private function activateLease(DrydockLease $lease) {
999 $resource = $lease->getResource();
1000 if (!$resource) {
1001 throw new Exception(
1002 pht('Trying to activate lease with no resource.'));
1003 }
1004
1005 $resource_status = $resource->getStatus();
1006
1007 if ($resource_status == DrydockResourceStatus::STATUS_PENDING) {
1008 throw new PhabricatorWorkerYieldException(15);
1009 }
1010
1011 if ($resource_status != DrydockResourceStatus::STATUS_ACTIVE) {
1012 throw new DrydockAcquiredBrokenResourceException(
1013 pht(
1014 'Trying to activate lease ("%s") on a resource ("%s") in '.
1015 'the wrong status ("%s").',
1016 $lease->getPHID(),
1017 $resource->getPHID(),
1018 $resource_status));
1019 }
1020
1021 // NOTE: We can race resource destruction here. Between the time we
1022 // performed the read above and now, the resource might have closed, so
1023 // we may activate leases on dead resources. At least for now, this seems
1024 // fine: a resource dying right before we activate a lease on it should not
1025 // be distinguishable from a resource dying right after we activate a lease
1026 // on it. We end up with an active lease on a dead resource either way, and
1027 // can not prevent resources dying from lightning strikes.
1028
1029 $blueprint = $resource->getBlueprint();
1030 $blueprint->activateLease($resource, $lease);
1031 $this->validateActivatedLease($blueprint, $resource, $lease);
1032 }
1033
1034 /**
1035 * @task activate
1036 */
1037 private function validateActivatedLease(
1038 DrydockBlueprint $blueprint,
1039 DrydockResource $resource,
1040 DrydockLease $lease) {
1041
1042 if (!$lease->isActivatedLease()) {
1043 throw new Exception(
1044 pht(
1045 'Blueprint "%s" (of type "%s") is not properly implemented: it '.
1046 'returned from "%s" without activating a lease.',
1047 $blueprint->getBlueprintName(),
1048 $blueprint->getClassName(),
1049 'acquireLease()'));
1050 }
1051
1052 }
1053
1054
1055/* -( Releasing Leases )--------------------------------------------------- */
1056
1057
1058 /**
1059 * @task release
1060 */
1061 private function releaseLease(DrydockLease $lease) {
1062 $lease
1063 ->setStatus(DrydockLeaseStatus::STATUS_RELEASED)
1064 ->save();
1065
1066 $lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST);
1067
1068 $resource = $lease->getResource();
1069 if ($resource) {
1070 $blueprint = $resource->getBlueprint();
1071 $blueprint->didReleaseLease($resource, $lease);
1072 }
1073
1074 $this->destroyLease($lease);
1075 }
1076
1077
1078/* -( Breaking Leases )---------------------------------------------------- */
1079
1080
1081 /**
1082 * @task break
1083 */
1084 protected function breakLease(DrydockLease $lease, Exception $ex) {
1085 switch ($lease->getStatus()) {
1086 case DrydockLeaseStatus::STATUS_BROKEN:
1087 case DrydockLeaseStatus::STATUS_RELEASED:
1088 case DrydockLeaseStatus::STATUS_DESTROYED:
1089 throw new Exception(
1090 pht(
1091 'Unexpected failure while destroying lease ("%s").',
1092 $lease->getPHID()),
1093 0,
1094 $ex);
1095 }
1096
1097 $lease
1098 ->setStatus(DrydockLeaseStatus::STATUS_BROKEN)
1099 ->save();
1100
1101 $lease->logEvent(
1102 DrydockLeaseActivationFailureLogType::LOGCONST,
1103 array(
1104 'class' => get_class($ex),
1105 'message' => $ex->getMessage(),
1106 ));
1107
1108 $lease->awakenTasks();
1109
1110 $this->queueTask(
1111 self::class,
1112 array(
1113 'leasePHID' => $lease->getPHID(),
1114 ),
1115 array(
1116 'objectPHID' => $lease->getPHID(),
1117 ));
1118
1119 throw new PhabricatorWorkerPermanentFailureException(
1120 pht(
1121 'Permanent failure while activating lease ("%s"): %s',
1122 $lease->getPHID(),
1123 $ex->getMessage()));
1124 }
1125
1126
1127/* -( Destroying Leases )-------------------------------------------------- */
1128
1129
1130 /**
1131 * @task destroy
1132 */
1133 private function destroyLease(DrydockLease $lease) {
1134 $resource = $lease->getResource();
1135
1136 if ($resource) {
1137 $blueprint = $resource->getBlueprint();
1138 $blueprint->destroyLease($resource, $lease);
1139 }
1140
1141 DrydockSlotLock::releaseLocks($lease->getPHID());
1142
1143 $lease
1144 ->setStatus(DrydockLeaseStatus::STATUS_DESTROYED)
1145 ->save();
1146
1147 $lease->logEvent(DrydockLeaseDestroyedLogType::LOGCONST);
1148
1149 $lease->awakenTasks();
1150 }
1151
1152}