@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add staging area support to Harbormaster/Drydock + various fixes
Summary:
Ref T9252. This primarily allows Harbormaster to request (and Drydock to fulfill) working copies with a patch from a staging area. Doing this means we can do builds on in-review changes from `arc diff`.
This is a little cobbled-together but should basically work.
Also fix some other issues:
- Yielded, awakend workers are fine to update but could complain.
- We can't log slot lock failures to resources if we don't end up saving them.
- Killing the transaction would wipe out the log.
- Fix some TODOs, etc.
Test Plan: Ran Harbormaster builds on a local revision.
Reviewers: hach-que, chad
Reviewed By: chad
Maniphest Tasks: T9252
Differential Revision: https://secure.phabricator.com/D14214
+129
-42
+11
src/applications/differential/storage/DifferentialDiff.php
+11
src/applications/differential/storage/DifferentialDiff.php
···
446
446
$results['repository.phid'] = $repo->getPHID();
447
447
$results['repository.vcs'] = $repo->getVersionControlSystem();
448
448
$results['repository.uri'] = $repo->getPublicCloneURI();
449
+
450
+
// TODO: We're just hoping to get lucky. Instead, `arc` should store
451
+
// where it sent changes and we should only provide staging details
452
+
// if we reasonably believe they are accurate.
453
+
$staging_ref = 'refs/tags/phabricator/diff/'.$this->getID();
454
+
$results['repository.staging.uri'] = $repo->getStagingURI();
455
+
$results['repository.staging.ref'] = $staging_ref;
449
456
}
450
457
}
451
458
···
466
473
pht('The version control system, either "svn", "hg" or "git".'),
467
474
'repository.uri' =>
468
475
pht('The URI to clone or checkout the repository from.'),
476
+
'repository.staging.uri' =>
477
+
pht('The URI of the staging repository.'),
478
+
'repository.staging.ref' =>
479
+
pht('The ref name for this change in the staging repository.'),
469
480
);
470
481
}
471
482
+8
-4
src/applications/drydock/blueprint/DrydockBlueprintImplementation.php
+8
-4
src/applications/drydock/blueprint/DrydockBlueprintImplementation.php
···
295
295
$lease_status = $lease->getStatus();
296
296
297
297
switch ($lease_status) {
298
+
case DrydockLeaseStatus::STATUS_PENDING:
298
299
case DrydockLeaseStatus::STATUS_ACQUIRED:
299
-
// TODO: Temporary failure.
300
-
throw new Exception(pht('Lease still activating.'));
300
+
throw new PhabricatorWorkerYieldException(15);
301
301
case DrydockLeaseStatus::STATUS_ACTIVE:
302
302
return;
303
303
default:
304
-
// TODO: Permanent failure.
305
-
throw new Exception(pht('Lease in bad state.'));
304
+
throw new Exception(
305
+
pht(
306
+
'Lease ("%s") is in bad state ("%s"), expected "%s".',
307
+
$lease->getPHID(),
308
+
$lease_status,
309
+
DrydockLeaseStatus::STATUS_ACTIVE));
306
310
}
307
311
}
308
312
+16
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
+16
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
···
216
216
$commit = idx($spec, 'commit');
217
217
$branch = idx($spec, 'branch');
218
218
219
+
$ref = idx($spec, 'ref');
220
+
219
221
if ($commit !== null) {
220
222
$cmd[] = 'git reset --hard %s';
221
223
$arg[] = $commit;
···
225
227
226
228
$cmd[] = 'git reset --hard origin/%s';
227
229
$arg[] = $branch;
230
+
} else if ($ref) {
231
+
$ref_uri = $ref['uri'];
232
+
$ref_ref = $ref['ref'];
233
+
234
+
$cmd[] = 'git fetch --no-tags -- %s +%s:%s';
235
+
$arg[] = $ref_uri;
236
+
$arg[] = $ref_ref;
237
+
$arg[] = $ref_ref;
238
+
239
+
$cmd[] = 'git checkout %s';
240
+
$arg[] = $ref_ref;
241
+
242
+
$cmd[] = 'git reset --hard %s';
243
+
$arg[] = $ref_ref;
228
244
} else {
229
245
$cmd[] = 'git reset --hard HEAD';
230
246
}
+11
src/applications/drydock/storage/DrydockBlueprint.php
+11
src/applications/drydock/storage/DrydockBlueprint.php
···
107
107
return $this->fields;
108
108
}
109
109
110
+
public function logEvent($type, array $data = array()) {
111
+
$log = id(new DrydockLog())
112
+
->setEpoch(PhabricatorTime::getNow())
113
+
->setType($type)
114
+
->setData($data);
115
+
116
+
$log->setBlueprintPHID($this->getPHID());
117
+
118
+
return $log->save();
119
+
}
120
+
110
121
111
122
/* -( Allocating Resources )----------------------------------------------- */
112
123
+14
-11
src/applications/drydock/storage/DrydockLease.php
+14
-11
src/applications/drydock/storage/DrydockLease.php
···
233
233
234
234
$this->openTransaction();
235
235
try {
236
-
try {
237
-
DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
238
-
$this->slotLocks = array();
239
-
} catch (DrydockSlotLockException $ex) {
240
-
$this->logEvent(
241
-
DrydockSlotLockFailureLogType::LOGCONST,
242
-
array(
243
-
'locks' => $ex->getLockMap(),
244
-
));
245
-
throw $ex;
246
-
}
236
+
DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
237
+
$this->slotLocks = array();
238
+
} catch (DrydockSlotLockException $ex) {
239
+
$this->killTransaction();
240
+
241
+
$this->logEvent(
242
+
DrydockSlotLockFailureLogType::LOGCONST,
243
+
array(
244
+
'locks' => $ex->getLockMap(),
245
+
));
246
+
247
+
throw $ex;
248
+
}
247
249
250
+
try {
248
251
$this
249
252
->setResourcePHID($resource->getPHID())
250
253
->attachResource($resource)
+18
-11
src/applications/drydock/storage/DrydockResource.php
+18
-11
src/applications/drydock/storage/DrydockResource.php
···
148
148
}
149
149
150
150
$this->openTransaction();
151
+
151
152
try {
152
-
try {
153
-
DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
154
-
$this->slotLocks = array();
155
-
} catch (DrydockSlotLockException $ex) {
156
-
$this->logEvent(
157
-
DrydockSlotLockFailureLogType::LOGCONST,
158
-
array(
159
-
'locks' => $ex->getLockMap(),
160
-
));
161
-
throw $ex;
162
-
}
153
+
DrydockSlotLock::acquireLocks($this->getPHID(), $this->slotLocks);
154
+
$this->slotLocks = array();
155
+
} catch (DrydockSlotLockException $ex) {
156
+
$this->killTransaction();
157
+
158
+
// NOTE: We have to log this on the blueprint, as the resource is not
159
+
// going to be saved so the PHID will vanish.
160
+
$this->getBlueprint()->logEvent(
161
+
DrydockSlotLockFailureLogType::LOGCONST,
162
+
array(
163
+
'locks' => $ex->getLockMap(),
164
+
));
165
+
166
+
throw $ex;
167
+
}
163
168
169
+
try {
164
170
$this
165
171
->setStatus($new_status)
166
172
->save();
···
168
174
$this->killTransaction();
169
175
throw $ex;
170
176
}
177
+
171
178
$this->saveTransaction();
172
179
173
180
$this->isAllocated = true;
+4
-3
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
+4
-3
src/applications/drydock/worker/DrydockLeaseUpdateWorker.php
···
660
660
$lease->logEvent(DrydockLeaseReleasedLogType::LOGCONST);
661
661
662
662
$resource = $lease->getResource();
663
-
$blueprint = $resource->getBlueprint();
664
-
665
-
$blueprint->didReleaseLease($resource, $lease);
663
+
if ($resource) {
664
+
$blueprint = $resource->getBlueprint();
665
+
$blueprint->didReleaseLease($resource, $lease);
666
+
}
666
667
667
668
$this->destroyLease($lease);
668
669
}
+30
-4
src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php
+30
-4
src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php
···
113
113
$variables = $build_target->getVariables();
114
114
115
115
$repository_phid = idx($variables, 'repository.phid');
116
+
if (!$repository_phid) {
117
+
throw new Exception(
118
+
pht(
119
+
'Unable to determine how to clone the repository for this '.
120
+
'buildable: it is not associated with a tracked repository.'));
121
+
}
122
+
116
123
$also_phids = $build_target->getFieldValue('repositoryPHIDs');
117
124
118
125
$all_phids = $also_phids;
···
132
139
$phid));
133
140
}
134
141
}
135
-
136
-
$commit = idx($variables, 'repository.commit');
137
142
138
143
$map = array();
139
144
···
147
152
148
153
$repository = $repositories[$repository_phid];
149
154
155
+
$commit = idx($variables, 'repository.commit');
156
+
$ref_uri = idx($variables, 'repository.staging.uri');
157
+
$ref_ref = idx($variables, 'repository.staging.ref');
158
+
if ($commit) {
159
+
$spec = array(
160
+
'commit' => $commit,
161
+
);
162
+
} else if ($ref_uri && $ref_ref) {
163
+
$spec = array(
164
+
'ref' => array(
165
+
'uri' => $ref_uri,
166
+
'ref' => $ref_ref,
167
+
),
168
+
);
169
+
} else {
170
+
throw new Exception(
171
+
pht(
172
+
'Unable to determine how to fetch changes: this buildable does not '.
173
+
'identify a commit or a staging ref. You may need to configure a '.
174
+
'repository staging area.'));
175
+
}
176
+
150
177
$directory = $repository->getCloneName();
151
178
$map[$directory] = array(
152
179
'phid' => $repository->getPHID(),
153
-
'commit' => $commit,
154
180
'default' => true,
155
-
);
181
+
) + $spec;
156
182
157
183
return $map;
158
184
}
+17
-9
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
+17
-9
src/infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php
···
86
86
}
87
87
88
88
protected function checkLease() {
89
-
if ($this->leaseOwner) {
90
-
$current_server_time = $this->serverTime + (time() - $this->localTime);
91
-
if ($current_server_time >= $this->leaseExpires) {
92
-
throw new Exception(
93
-
pht(
94
-
'Trying to update Task %d (%s) after lease expiration!',
95
-
$this->getID(),
96
-
$this->getTaskClass()));
97
-
}
89
+
$owner = $this->leaseOwner;
90
+
91
+
if (!$owner) {
92
+
return;
93
+
}
94
+
95
+
if ($owner == PhabricatorWorker::YIELD_OWNER) {
96
+
return;
97
+
}
98
+
99
+
$current_server_time = $this->serverTime + (time() - $this->localTime);
100
+
if ($current_server_time >= $this->leaseExpires) {
101
+
throw new Exception(
102
+
pht(
103
+
'Trying to update Task %d (%s) after lease expiration!',
104
+
$this->getID(),
105
+
$this->getTaskClass()));
98
106
}
99
107
}
100
108