@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
1<?php
2
3final class DrydockWorkingCopyBlueprintImplementation
4 extends DrydockBlueprintImplementation {
5
6 const PHASE_SQUASHMERGE = 'squashmerge';
7 const PHASE_REMOTEFETCH = 'blueprint.workingcopy.fetch.remote';
8 const PHASE_MERGEFETCH = 'blueprint.workingcopy.fetch.staging';
9
10 public function isEnabled() {
11 return true;
12 }
13
14 public function getBlueprintName() {
15 return pht('Working Copy');
16 }
17
18 public function getBlueprintIcon() {
19 return 'fa-folder-open';
20 }
21
22 public function getDescription() {
23 return pht('Allows Drydock to check out working copies of repositories.');
24 }
25
26 public function canAnyBlueprintEverAllocateResourceForLease(
27 DrydockLease $lease) {
28 return true;
29 }
30
31 public function canEverAllocateResourceForLease(
32 DrydockBlueprint $blueprint,
33 DrydockLease $lease) {
34 return true;
35 }
36
37 public function canAllocateResourceForLease(
38 DrydockBlueprint $blueprint,
39 DrydockLease $lease) {
40 $viewer = $this->getViewer();
41
42 if ($this->shouldLimitAllocatingPoolSize($blueprint)) {
43 return false;
44 }
45
46 return true;
47 }
48
49 public function canAcquireLeaseOnResource(
50 DrydockBlueprint $blueprint,
51 DrydockResource $resource,
52 DrydockLease $lease) {
53
54 // Don't hand out leases on working copies which have not activated, since
55 // it may take an arbitrarily long time for them to acquire a host.
56 if (!$resource->isActive()) {
57 return false;
58 }
59
60 $need_map = $lease->getAttribute('repositories.map');
61 if (!is_array($need_map)) {
62 return false;
63 }
64
65 $have_map = $resource->getAttribute('repositories.map');
66 if (!is_array($have_map)) {
67 return false;
68 }
69
70 $have_as = ipull($have_map, 'phid');
71 $need_as = ipull($need_map, 'phid');
72
73 foreach ($need_as as $need_directory => $need_phid) {
74 if (empty($have_as[$need_directory])) {
75 // This resource is missing a required working copy.
76 return false;
77 }
78
79 if ($have_as[$need_directory] != $need_phid) {
80 // This resource has a required working copy, but it contains
81 // the wrong repository.
82 return false;
83 }
84
85 unset($have_as[$need_directory]);
86 }
87
88 if ($have_as && $lease->getAttribute('repositories.strict')) {
89 // This resource has extra repositories, but the lease is strict about
90 // which repositories are allowed to exist.
91 return false;
92 }
93
94 if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) {
95 return false;
96 }
97
98 return true;
99 }
100
101 public function acquireLease(
102 DrydockBlueprint $blueprint,
103 DrydockResource $resource,
104 DrydockLease $lease) {
105
106 $lease
107 ->needSlotLock($this->getLeaseSlotLock($resource))
108 ->acquireOnResource($resource);
109 }
110
111 private function getLeaseSlotLock(DrydockResource $resource) {
112 $resource_phid = $resource->getPHID();
113 return "workingcopy.lease({$resource_phid})";
114 }
115
116 public function allocateResource(
117 DrydockBlueprint $blueprint,
118 DrydockLease $lease) {
119
120 $resource = $this->newResourceTemplate($blueprint);
121
122 $resource_phid = $resource->getPHID();
123
124 $blueprint_phids = $blueprint->getFieldValue('blueprintPHIDs');
125
126 $host_lease = $this->newLease($blueprint)
127 ->setResourceType('host')
128 ->setOwnerPHID($resource_phid)
129 ->setAttribute('workingcopy.resourcePHID', $resource_phid)
130 ->setAllowedBlueprintPHIDs($blueprint_phids);
131 $resource->setAttribute('host.leasePHID', $host_lease->getPHID());
132
133 $map = $this->getWorkingCopyRepositoryMap($lease);
134 $resource->setAttribute('repositories.map', $map);
135
136 $slot_lock = $this->getConcurrentResourceLimitSlotLock($blueprint);
137 if ($slot_lock !== null) {
138 $resource->needSlotLock($slot_lock);
139 }
140
141 $resource->allocateResource();
142
143 $host_lease->queueForActivation();
144
145 return $resource;
146 }
147
148 private function getWorkingCopyRepositoryMap(DrydockLease $lease) {
149 $attribute = 'repositories.map';
150 $map = $lease->getAttribute($attribute);
151
152 // TODO: Leases should validate their attributes more formally.
153
154 if (!is_array($map) || !$map) {
155 $message = array();
156 if ($map === null) {
157 $message[] = pht(
158 'Working copy lease is missing required attribute "%s".',
159 $attribute);
160 } else {
161 $message[] = pht(
162 'Working copy lease has invalid attribute "%s".',
163 $attribute);
164 }
165
166 $message[] = pht(
167 'Attribute "repositories.map" should be a map of repository '.
168 'specifications.');
169
170 $message = implode("\n\n", $message);
171
172 throw new Exception($message);
173 }
174
175 foreach ($map as $key => $value) {
176 $map[$key] = array_select_keys(
177 $value,
178 array(
179 'phid',
180 ));
181 }
182
183 return $map;
184 }
185
186 public function activateResource(
187 DrydockBlueprint $blueprint,
188 DrydockResource $resource) {
189
190 $lease = $this->loadHostLease($resource);
191 $this->requireActiveLease($lease);
192
193 $command_type = DrydockCommandInterface::INTERFACE_TYPE;
194 $interface = $lease->getInterface($command_type);
195
196 // TODO: Make this configurable.
197 $resource_id = $resource->getID();
198 $root = "/var/drydock/workingcopy-{$resource_id}";
199
200 $map = $resource->getAttribute('repositories.map');
201
202 $futures = array();
203 $repositories = $this->loadRepositories(ipull($map, 'phid'));
204 foreach ($map as $directory => $spec) {
205 // TODO: Validate directory isn't goofy like "/etc" or "../../lol"
206 // somewhere?
207
208 $repository = $repositories[$spec['phid']];
209 $path = "{$root}/repo/{$directory}/";
210
211 $future = $interface->getExecFuture(
212 'git clone -- %s %s',
213 (string)$repository->getCloneURIObject(),
214 $path);
215
216 $future->setTimeout($repository->getEffectiveCopyTimeLimit());
217
218 $futures[$directory] = $future;
219 }
220
221 foreach (new FutureIterator($futures) as $key => $future) {
222 $future->resolvex();
223 }
224
225 $resource
226 ->setAttribute('workingcopy.root', $root)
227 ->activateResource();
228 }
229
230 public function destroyResource(
231 DrydockBlueprint $blueprint,
232 DrydockResource $resource) {
233
234 try {
235 $lease = $this->loadHostLease($resource);
236 } catch (Exception $ex) {
237 // If we can't load the lease, assume we don't need to take any actions
238 // to destroy it.
239 return;
240 }
241
242 // Destroy the lease on the host.
243 $lease->setReleaseOnDestruction(true);
244
245 if ($lease->isActive()) {
246 // Destroy the working copy on disk.
247 $command_type = DrydockCommandInterface::INTERFACE_TYPE;
248 $interface = $lease->getInterface($command_type);
249
250 $root_key = 'workingcopy.root';
251 $root = $resource->getAttribute($root_key);
252 if (strlen($root)) {
253 $interface->execx('rm -rf -- %s', $root);
254 }
255 }
256 }
257
258 public function getResourceName(
259 DrydockBlueprint $blueprint,
260 DrydockResource $resource) {
261 return pht('Working Copy');
262 }
263
264
265 public function activateLease(
266 DrydockBlueprint $blueprint,
267 DrydockResource $resource,
268 DrydockLease $lease) {
269
270 $host_lease = $this->loadHostLease($resource);
271 $command_type = DrydockCommandInterface::INTERFACE_TYPE;
272 $interface = $host_lease->getInterface($command_type);
273
274 $map = $lease->getAttribute('repositories.map');
275 $root = $resource->getAttribute('workingcopy.root');
276
277 $repositories = $this->loadRepositories(ipull($map, 'phid'));
278
279 $default = null;
280 foreach ($map as $directory => $spec) {
281 $repository = $repositories[$spec['phid']];
282
283 $interface->pushWorkingDirectory("{$root}/repo/{$directory}/");
284
285 $cmd = array();
286 $arg = array();
287
288 $cmd[] = 'git clean -d --force';
289 $cmd[] = 'git fetch';
290
291 $commit = idx($spec, 'commit');
292 $branch = idx($spec, 'branch');
293
294 $ref = idx($spec, 'ref');
295
296 // Reset things first, in case previous builds left anything staged or
297 // dirty. Note that we don't reset to "HEAD" because that does not work
298 // in empty repositories.
299 $cmd[] = 'git reset --hard';
300
301 if ($commit !== null) {
302 $cmd[] = 'git checkout %s --';
303 $arg[] = $commit;
304 } else if ($branch !== null) {
305 $cmd[] = 'git checkout %s --';
306 $arg[] = $branch;
307
308 $cmd[] = 'git reset --hard origin/%s';
309 $arg[] = $branch;
310 }
311
312 $this->newExecvFuture($interface, $cmd, $arg)
313 ->setTimeout($repository->getEffectiveCopyTimeLimit())
314 ->resolvex();
315
316 if (idx($spec, 'default')) {
317 $default = $directory;
318 }
319
320 // If we're fetching a ref from a remote, do that separately so we can
321 // raise a more tailored error.
322 if ($ref) {
323 $cmd = array();
324 $arg = array();
325
326 $ref_uri = $ref['uri'];
327 $ref_ref = $ref['ref'];
328
329 $cmd[] = 'git fetch --no-tags -- %s +%s:%s';
330 $arg[] = $ref_uri;
331 $arg[] = $ref_ref;
332 $arg[] = $ref_ref;
333
334 $cmd[] = 'git checkout %s --';
335 $arg[] = $ref_ref;
336
337 try {
338 $this->newExecvFuture($interface, $cmd, $arg)
339 ->setTimeout($repository->getEffectiveCopyTimeLimit())
340 ->resolvex();
341 } catch (CommandException $ex) {
342 $display_command = csprintf(
343 'git fetch %R %R',
344 $ref_uri,
345 $ref_ref);
346
347 $error = DrydockCommandError::newFromCommandException($ex)
348 ->setPhase(self::PHASE_REMOTEFETCH)
349 ->setDisplayCommand($display_command);
350
351 $lease->setAttribute(
352 'workingcopy.vcs.error',
353 $error->toDictionary());
354
355 throw $ex;
356 }
357 }
358
359 $merges = idx($spec, 'merges');
360 if ($merges) {
361 foreach ($merges as $merge) {
362 $this->applyMerge($lease, $interface, $merge);
363 }
364 }
365
366 $interface->popWorkingDirectory();
367 }
368
369 if ($default === null) {
370 $default = head_key($map);
371 }
372
373 // TODO: Use working storage?
374 $lease->setAttribute('workingcopy.default', "{$root}/repo/{$default}/");
375
376 $lease->activateOnResource($resource);
377 }
378
379 public function didReleaseLease(
380 DrydockBlueprint $blueprint,
381 DrydockResource $resource,
382 DrydockLease $lease) {
383 // We leave working copies around even if there are no leases on them,
384 // since the cost to maintain them is nearly zero but rebuilding them is
385 // moderately expensive and it's likely that they'll be reused.
386 return;
387 }
388
389 public function destroyLease(
390 DrydockBlueprint $blueprint,
391 DrydockResource $resource,
392 DrydockLease $lease) {
393 // When we activate a lease we just reset the working copy state and do
394 // not create any new state, so we don't need to do anything special when
395 // destroying a lease.
396 return;
397 }
398
399 public function getType() {
400 return 'working-copy';
401 }
402
403 public function getInterface(
404 DrydockBlueprint $blueprint,
405 DrydockResource $resource,
406 DrydockLease $lease,
407 $type) {
408
409 switch ($type) {
410 case DrydockCommandInterface::INTERFACE_TYPE:
411 $host_lease = $this->loadHostLease($resource);
412 $command_interface = $host_lease->getInterface($type);
413
414 $path = $lease->getAttribute('workingcopy.default');
415 $command_interface->pushWorkingDirectory($path);
416
417 return $command_interface;
418 }
419 }
420
421 private function loadRepositories(array $phids) {
422 $viewer = $this->getViewer();
423
424 $repositories = id(new PhabricatorRepositoryQuery())
425 ->setViewer($viewer)
426 ->withPHIDs($phids)
427 ->execute();
428 $repositories = mpull($repositories, null, 'getPHID');
429
430 foreach ($phids as $phid) {
431 if (empty($repositories[$phid])) {
432 throw new Exception(
433 pht(
434 'Repository PHID "%s" does not exist.',
435 $phid));
436 }
437 }
438
439 foreach ($repositories as $repository) {
440 $repository_vcs = $repository->getVersionControlSystem();
441 switch ($repository_vcs) {
442 case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
443 break;
444 default:
445 throw new Exception(
446 pht(
447 'Repository ("%s") has unsupported VCS ("%s").',
448 $repository->getPHID(),
449 $repository_vcs));
450 }
451 }
452
453 return $repositories;
454 }
455
456 private function loadHostLease(DrydockResource $resource) {
457 $viewer = $this->getViewer();
458
459 $lease_phid = $resource->getAttribute('host.leasePHID');
460
461 $lease = id(new DrydockLeaseQuery())
462 ->setViewer($viewer)
463 ->withPHIDs(array($lease_phid))
464 ->executeOne();
465 if (!$lease) {
466 throw new Exception(
467 pht(
468 'Unable to load lease ("%s").',
469 $lease_phid));
470 }
471
472 return $lease;
473 }
474
475 protected function getCustomFieldSpecifications() {
476 return array(
477 'blueprintPHIDs' => array(
478 'name' => pht('Use Blueprints'),
479 'type' => 'blueprints',
480 'required' => true,
481 ),
482 );
483 }
484
485 protected function shouldUseConcurrentResourceLimit() {
486 return true;
487 }
488
489 private function applyMerge(
490 DrydockLease $lease,
491 DrydockCommandInterface $interface,
492 array $merge) {
493
494 $src_uri = $merge['src.uri'];
495 $src_ref = $merge['src.ref'];
496
497
498 try {
499 $interface->execx(
500 'git fetch --no-tags -- %s +%s:%s',
501 $src_uri,
502 $src_ref,
503 $src_ref);
504 } catch (CommandException $ex) {
505 $display_command = csprintf(
506 'git fetch %R +%R:%R',
507 $src_uri,
508 $src_ref,
509 $src_ref);
510
511 $error = DrydockCommandError::newFromCommandException($ex)
512 ->setPhase(self::PHASE_MERGEFETCH)
513 ->setDisplayCommand($display_command);
514
515 $lease->setAttribute('workingcopy.vcs.error', $error->toDictionary());
516
517 throw $ex;
518 }
519
520
521 // NOTE: This can never actually generate a commit because we pass
522 // "--squash", but git sometimes runs code to check that a username and
523 // email are configured anyway.
524 $real_command = csprintf(
525 'git -c user.name=%s -c user.email=%s merge --no-stat --squash -- %R',
526 'drydock',
527 'drydock@phabricator',
528 $src_ref);
529
530 try {
531 $interface->execx('%C', $real_command);
532 } catch (CommandException $ex) {
533 $display_command = csprintf(
534 'git merge --squash %R',
535 $src_ref);
536
537 $error = DrydockCommandError::newFromCommandException($ex)
538 ->setPhase(self::PHASE_SQUASHMERGE)
539 ->setDisplayCommand($display_command);
540
541 $lease->setAttribute('workingcopy.vcs.error', $error->toDictionary());
542 throw $ex;
543 }
544 }
545
546 public function getCommandError(DrydockLease $lease) {
547 return $lease->getAttribute('workingcopy.vcs.error');
548 }
549
550 private function execxv(
551 DrydockCommandInterface $interface,
552 array $commands,
553 array $arguments) {
554 return $this->newExecvFuture($interface, $commands, $arguments)->resolvex();
555 }
556
557 private function newExecvFuture(
558 DrydockCommandInterface $interface,
559 array $commands,
560 array $arguments) {
561
562 $commands = implode(' && ', $commands);
563 $argv = array_merge(array($commands), $arguments);
564
565 return call_user_func_array(array($interface, 'getExecFuture'), $argv);
566 }
567
568}