@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Allow WorkingCopy resources to have multiple working copies
Summary:
Ref T9252. For building Phabricator itself, we need to have `libphutil/`, `arcanist/` and `phabricator/` next to one another on disk.
Expand the Drydock WorkingCopy resource so that it can have multiple repositories if the caller needs them.
I'm not sure if I'm going to put the actual config for this in Harbormaster or Drydock yet, but the WorkingCopy resource itself should work the same way in either case.
Test Plan: Restarted a Harbormaster build which leases a working copy, saw it build as expected.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T9252
Differential Revision: https://secure.phabricator.com/D14180
+155
-64
+122
-57
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
+122
-57
src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
···
37
37
DrydockResource $resource,
38
38
DrydockLease $lease) {
39
39
40
-
$have_phid = $resource->getAttribute('repositoryPHID');
41
-
$need_phid = $lease->getAttribute('repositoryPHID');
40
+
$need_map = $lease->getAttribute('repositories.map');
41
+
if (!is_array($need_map)) {
42
+
return false;
43
+
}
44
+
45
+
$have_map = $resource->getAttribute('repositories.map');
46
+
if (!is_array($have_map)) {
47
+
return false;
48
+
}
49
+
50
+
$have_as = ipull($have_map, 'phid');
51
+
$need_as = ipull($need_map, 'phid');
52
+
53
+
foreach ($need_as as $need_directory => $need_phid) {
54
+
if (empty($have_as[$need_directory])) {
55
+
// This resource is missing a required working copy.
56
+
return false;
57
+
}
58
+
59
+
if ($have_as[$need_directory] != $need_phid) {
60
+
// This resource has a required working copy, but it contains
61
+
// the wrong repository.
62
+
return false;
63
+
}
42
64
43
-
if ($need_phid !== $have_phid) {
65
+
unset($have_as[$need_directory]);
66
+
}
67
+
68
+
if ($have_as && $lease->getAttribute('repositories.strict')) {
69
+
// This resource has extra repositories, but the lease is strict about
70
+
// which repositories are allowed to exist.
44
71
return false;
45
72
}
46
73
···
70
97
DrydockBlueprint $blueprint,
71
98
DrydockLease $lease) {
72
99
73
-
$repository_phid = $lease->getAttribute('repositoryPHID');
74
-
$repository = $this->loadRepository($repository_phid);
75
-
76
100
$resource = $this->newResourceTemplate(
77
101
$blueprint,
78
-
pht(
79
-
'Working Copy (%s)',
80
-
$repository->getCallsign()));
102
+
pht('Working Copy'));
81
103
82
104
$resource_phid = $resource->getPHID();
83
105
···
90
112
// TODO: Add some limits to the number of working copies we can have at
91
113
// once?
92
114
115
+
$map = $lease->getAttribute('repositories.map');
116
+
foreach ($map as $key => $value) {
117
+
$map[$key] = array_select_keys(
118
+
$value,
119
+
array(
120
+
'phid',
121
+
));
122
+
}
123
+
93
124
return $resource
94
-
->setAttribute('repositoryPHID', $repository->getPHID())
125
+
->setAttribute('repositories.map', $map)
95
126
->setAttribute('host.leasePHID', $host_lease->getPHID())
96
127
->allocateResource();
97
128
}
···
103
134
$lease = $this->loadHostLease($resource);
104
135
$this->requireActiveLease($lease);
105
136
106
-
$repository_phid = $resource->getAttribute('repositoryPHID');
107
-
$repository = $this->loadRepository($repository_phid);
108
-
$repository_id = $repository->getID();
109
-
110
137
$command_type = DrydockCommandInterface::INTERFACE_TYPE;
111
138
$interface = $lease->getInterface($command_type);
112
139
113
140
// TODO: Make this configurable.
114
141
$resource_id = $resource->getID();
115
142
$root = "/var/drydock/workingcopy-{$resource_id}";
116
-
$path = "{$root}/repo/{$repository_id}/";
143
+
144
+
$map = $resource->getAttribute('repositories.map');
145
+
146
+
$repositories = $this->loadRepositories(ipull($map, 'phid'));
147
+
foreach ($map as $directory => $spec) {
148
+
// TODO: Validate directory isn't goofy like "/etc" or "../../lol"
149
+
// somewhere?
150
+
151
+
$repository = $repositories[$spec['phid']];
152
+
$path = "{$root}/repo/{$directory}/";
117
153
118
-
$interface->execx(
119
-
'git clone -- %s %s',
120
-
(string)$repository->getCloneURIObject(),
121
-
$path);
154
+
// TODO: Run these in parallel?
155
+
$interface->execx(
156
+
'git clone -- %s %s',
157
+
(string)$repository->getCloneURIObject(),
158
+
$path);
159
+
}
122
160
123
161
$resource
124
162
->setAttribute('workingcopy.root', $root)
125
-
->setAttribute('workingcopy.path', $path)
126
163
->activateResource();
127
164
}
128
165
···
151
188
DrydockResource $resource,
152
189
DrydockLease $lease) {
153
190
191
+
$host_lease = $this->loadHostLease($resource);
154
192
$command_type = DrydockCommandInterface::INTERFACE_TYPE;
155
-
$interface = $lease->getInterface($command_type);
193
+
$interface = $host_lease->getInterface($command_type);
194
+
195
+
$map = $lease->getAttribute('repositories.map');
196
+
$root = $resource->getAttribute('workingcopy.root');
197
+
198
+
$default = null;
199
+
foreach ($map as $directory => $spec) {
200
+
$cmd = array();
201
+
$arg = array();
202
+
203
+
$cmd[] = 'cd %s';
204
+
$arg[] = "{$root}/repo/{$directory}/";
156
205
157
-
$cmd = array();
158
-
$arg = array();
206
+
$cmd[] = 'git clean -d --force';
207
+
$cmd[] = 'git fetch';
208
+
209
+
$commit = idx($spec, 'commit');
210
+
$branch = idx($spec, 'branch');
211
+
212
+
if ($commit !== null) {
213
+
$cmd[] = 'git reset --hard %s';
214
+
$arg[] = $commit;
215
+
} else if ($branch !== null) {
216
+
$cmd[] = 'git reset --hard %s';
217
+
$arg[] = $branch;
218
+
} else {
219
+
$cmd[] = 'git reset --hard HEAD';
220
+
}
159
221
160
-
$cmd[] = 'git clean -d --force';
161
-
$cmd[] = 'git reset --hard HEAD';
162
-
$cmd[] = 'git fetch';
222
+
$cmd = implode(' && ', $cmd);
223
+
$argv = array_merge(array($cmd), $arg);
163
224
164
-
$commit = $lease->getAttribute('commit');
165
-
$branch = $lease->getAttribute('branch');
225
+
$result = call_user_func_array(
226
+
array($interface, 'execx'),
227
+
$argv);
166
228
167
-
if ($commit !== null) {
168
-
$cmd[] = 'git reset --hard %s';
169
-
$arg[] = $commit;
170
-
} else if ($branch !== null) {
171
-
$cmd[] = 'git reset --hard %s';
172
-
$arg[] = $branch;
229
+
if (idx($spec, 'default')) {
230
+
$default = $directory;
231
+
}
173
232
}
174
233
175
-
$cmd = implode(' && ', $cmd);
176
-
$argv = array_merge(array($cmd), $arg);
234
+
if ($default === null) {
235
+
$default = head_key($map);
236
+
}
177
237
178
-
$result = call_user_func_array(
179
-
array($interface, 'execx'),
180
-
$argv);
238
+
// TODO: Use working storage?
239
+
$lease->setAttribute('workingcopy.default', "{$root}/repo/{$default}/");
181
240
182
241
$lease->activateOnResource($resource);
183
242
}
···
217
276
$host_lease = $this->loadHostLease($resource);
218
277
$command_interface = $host_lease->getInterface($type);
219
278
220
-
$path = $resource->getAttribute('workingcopy.path');
279
+
$path = $lease->getAttribute('workingcopy.default');
221
280
$command_interface->setWorkingDirectory($path);
222
281
223
282
return $command_interface;
224
283
}
225
284
}
226
285
227
-
private function loadRepository($repository_phid) {
228
-
$repository = id(new PhabricatorRepositoryQuery())
286
+
private function loadRepositories(array $phids) {
287
+
$repositories = id(new PhabricatorRepositoryQuery())
229
288
->setViewer(PhabricatorUser::getOmnipotentUser())
230
-
->withPHIDs(array($repository_phid))
231
-
->executeOne();
232
-
if (!$repository) {
233
-
// TODO: Permanent failure.
234
-
throw new Exception(
235
-
pht(
236
-
'Repository PHID "%s" does not exist.',
237
-
$repository_phid));
238
-
}
289
+
->withPHIDs($phids)
290
+
->execute();
291
+
$repositories = mpull($repositories, null, 'getPHID');
239
292
240
-
switch ($repository->getVersionControlSystem()) {
241
-
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
242
-
break;
243
-
default:
293
+
foreach ($phids as $phid) {
294
+
if (empty($repositories[$phid])) {
244
295
// TODO: Permanent failure.
245
-
throw new Exception(pht('Unsupported VCS!'));
296
+
throw new Exception(
297
+
pht(
298
+
'Repository PHID "%s" does not exist.',
299
+
$phid));
300
+
}
301
+
}
302
+
303
+
foreach ($repositories as $repository) {
304
+
switch ($repository->getVersionControlSystem()) {
305
+
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
306
+
break;
307
+
default:
308
+
// TODO: Permanent failure.
309
+
throw new Exception(pht('Unsupported VCS!'));
310
+
}
246
311
}
247
312
248
-
return $repository;
313
+
return $repositories;
249
314
}
250
315
251
316
private function loadHostLease(DrydockResource $resource) {
+33
-7
src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php
+33
-7
src/applications/harbormaster/step/HarbormasterLeaseWorkingCopyBuildStepImplementation.php
···
45
45
->setResourceType($working_copy_type)
46
46
->setOwnerPHID($build_target->getPHID());
47
47
48
-
$variables = $build_target->getVariables();
48
+
$map = $this->buildRepositoryMap($build_target);
49
49
50
-
$repository_phid = idx($variables, 'repository.phid');
51
-
$commit = idx($variables, 'repository.commit');
52
-
53
-
$lease
54
-
->setAttribute('repositoryPHID', $repository_phid)
55
-
->setAttribute('commit', $commit);
50
+
$lease->setAttribute('repositories.map', $map);
56
51
57
52
$task_id = $this->getCurrentWorkerTaskID();
58
53
if ($task_id) {
···
106
101
'required' => true,
107
102
),
108
103
);
104
+
}
105
+
106
+
private function buildRepositoryMap(HarbormasterBuildTarget $build_target) {
107
+
$viewer = PhabricatorUser::getOmnipotentUser();
108
+
$variables = $build_target->getVariables();
109
+
110
+
$repository_phid = idx($variables, 'repository.phid');
111
+
112
+
$repository = id(new PhabricatorRepositoryQuery())
113
+
->setViewer($viewer)
114
+
->withPHIDs(array($repository_phid))
115
+
->executeOne();
116
+
if (!$repository) {
117
+
throw new PhabricatorWorkerPermanentFailureException(
118
+
pht(
119
+
'Unable to load repository with PHID "%s".',
120
+
$repository_phid));
121
+
}
122
+
123
+
$commit = idx($variables, 'repository.commit');
124
+
125
+
$map = array();
126
+
127
+
$directory = $repository->getCloneName();
128
+
$map[$directory] = array(
129
+
'phid' => $repository->getPHID(),
130
+
'commit' => $commit,
131
+
'default' => true,
132
+
);
133
+
134
+
return $map;
109
135
}
110
136
111
137
}