@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add some Drydock documentation plus "Test Configuration" for repository automation
Summary:
Ref T182. Ref T9252.
- Adds a "Test" repository operation that just runs `git status` to see if things work.
- Adds a button for it in Edit Repository.
- Shows operation status on the operation detail view to make this workflow work a little better.
- Adds a lot of words. Words words words words.
Test Plan:
- Tested repository operation.
- Read words.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T182, T9252
Differential Revision: https://secure.phabricator.com/D14349
authored by
epriestley
and committed by
epriestley
a763f951
cea633f6
+776
-10
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
709
709
'DiffusionRepositoryRemarkupRule' => 'applications/diffusion/remarkup/DiffusionRepositoryRemarkupRule.php',
710
710
'DiffusionRepositorySymbolsController' => 'applications/diffusion/controller/DiffusionRepositorySymbolsController.php',
711
711
'DiffusionRepositoryTag' => 'applications/diffusion/data/DiffusionRepositoryTag.php',
712
+
'DiffusionRepositoryTestAutomationController' => 'applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php',
712
713
'DiffusionRequest' => 'applications/diffusion/request/DiffusionRequest.php',
713
714
'DiffusionResolveRefsConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionResolveRefsConduitAPIMethod.php',
714
715
'DiffusionResolveUserQuery' => 'applications/diffusion/query/DiffusionResolveUserQuery.php',
···
909
910
'DrydockSlotLock' => 'applications/drydock/storage/DrydockSlotLock.php',
910
911
'DrydockSlotLockException' => 'applications/drydock/exception/DrydockSlotLockException.php',
911
912
'DrydockSlotLockFailureLogType' => 'applications/drydock/logtype/DrydockSlotLockFailureLogType.php',
913
+
'DrydockTestRepositoryOperation' => 'applications/drydock/operation/DrydockTestRepositoryOperation.php',
912
914
'DrydockWebrootInterface' => 'applications/drydock/interface/webroot/DrydockWebrootInterface.php',
913
915
'DrydockWorker' => 'applications/drydock/worker/DrydockWorker.php',
914
916
'DrydockWorkingCopyBlueprintImplementation' => 'applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php',
···
4465
4467
'DiffusionRepositoryRemarkupRule' => 'PhabricatorObjectRemarkupRule',
4466
4468
'DiffusionRepositorySymbolsController' => 'DiffusionRepositoryEditController',
4467
4469
'DiffusionRepositoryTag' => 'Phobject',
4470
+
'DiffusionRepositoryTestAutomationController' => 'DiffusionRepositoryEditController',
4468
4471
'DiffusionRequest' => 'Phobject',
4469
4472
'DiffusionResolveRefsConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod',
4470
4473
'DiffusionResolveUserQuery' => 'Phobject',
···
4705
4708
'DrydockSlotLock' => 'DrydockDAO',
4706
4709
'DrydockSlotLockException' => 'Exception',
4707
4710
'DrydockSlotLockFailureLogType' => 'DrydockLogType',
4711
+
'DrydockTestRepositoryOperation' => 'DrydockRepositoryOperationType',
4708
4712
'DrydockWebrootInterface' => 'DrydockInterface',
4709
4713
'DrydockWorker' => 'PhabricatorWorker',
4710
4714
'DrydockWorkingCopyBlueprintImplementation' => 'DrydockBlueprintImplementation',
+1
src/applications/diffusion/application/PhabricatorDiffusionApplication.php
+1
src/applications/diffusion/application/PhabricatorDiffusionApplication.php
···
103
103
'symbol/' => 'DiffusionRepositorySymbolsController',
104
104
'staging/' => 'DiffusionRepositoryEditStagingController',
105
105
'automation/' => 'DiffusionRepositoryEditAutomationController',
106
+
'testautomation/' => 'DiffusionRepositoryTestAutomationController',
106
107
),
107
108
'pathtree/(?P<dblob>.*)' => 'DiffusionPathTreeController',
108
109
'mirror/' => array(
+1
-1
src/applications/diffusion/controller/DiffusionRepositoryEditAutomationController.php
+1
-1
src/applications/diffusion/controller/DiffusionRepositoryEditAutomationController.php
+13
src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php
+13
src/applications/diffusion/controller/DiffusionRepositoryEditMainController.php
···
688
688
$this->getRepositoryControllerURI($repository, 'edit/automation/'));
689
689
$view->addAction($edit);
690
690
691
+
$can_test = $repository->canPerformAutomation();
692
+
693
+
$test = id(new PhabricatorActionView())
694
+
->setIcon('fa-gamepad')
695
+
->setName(pht('Test Configuration'))
696
+
->setWorkflow(true)
697
+
->setDisabled(!$can_test)
698
+
->setHref(
699
+
$this->getRepositoryControllerURI(
700
+
$repository,
701
+
'edit/testautomation/'));
702
+
$view->addAction($test);
703
+
691
704
return $view;
692
705
}
693
706
+78
src/applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php
+78
src/applications/diffusion/controller/DiffusionRepositoryTestAutomationController.php
···
1
+
<?php
2
+
3
+
final class DiffusionRepositoryTestAutomationController
4
+
extends DiffusionRepositoryEditController {
5
+
6
+
protected function processDiffusionRequest(AphrontRequest $request) {
7
+
$viewer = $this->getViewer();
8
+
$drequest = $this->diffusionRequest;
9
+
$repository = $drequest->getRepository();
10
+
11
+
$repository = id(new PhabricatorRepositoryQuery())
12
+
->setViewer($viewer)
13
+
->requireCapabilities(
14
+
array(
15
+
PhabricatorPolicyCapability::CAN_VIEW,
16
+
PhabricatorPolicyCapability::CAN_EDIT,
17
+
))
18
+
->withIDs(array($repository->getID()))
19
+
->executeOne();
20
+
if (!$repository) {
21
+
return new Aphront404Response();
22
+
}
23
+
24
+
$edit_uri = $this->getRepositoryControllerURI($repository, 'edit/');
25
+
26
+
if (!$repository->canPerformAutomation()) {
27
+
return $this->newDialog()
28
+
->setTitle(pht('Automation Not Configured'))
29
+
->appendParagraph(
30
+
pht(
31
+
'You can not run a configuration test for this repository '.
32
+
'because you have not configured repository automation yet. '.
33
+
'Configure it first, then test the configuration.'))
34
+
->addCancelButton($edit_uri);
35
+
}
36
+
37
+
if ($request->isFormPost()) {
38
+
$op = new DrydockTestRepositoryOperation();
39
+
40
+
$operation = DrydockRepositoryOperation::initializeNewOperation($op)
41
+
->setAuthorPHID($viewer->getPHID())
42
+
->setObjectPHID($repository->getPHID())
43
+
->setRepositoryPHID($repository->getPHID())
44
+
->setRepositoryTarget('none:')
45
+
->save();
46
+
47
+
$operation->scheduleUpdate();
48
+
49
+
$operation_id = $operation->getID();
50
+
$operation_uri = "/drydock/operation/{$operation_id}/";
51
+
52
+
return id(new AphrontRedirectResponse())
53
+
->setURI($operation_uri);
54
+
}
55
+
56
+
return $this->newDialog()
57
+
->setTitle(pht('Test Automation Configuration'))
58
+
->appendParagraph(
59
+
pht(
60
+
'This configuration test will build a working copy of the '.
61
+
'repository and perform some basic validation. If it works, '.
62
+
'your configuration is substantially correct.'))
63
+
->appendParagraph(
64
+
pht(
65
+
'The test will not perform any writes against the repository, so '.
66
+
'write operations may still fail even if the test passes. This '.
67
+
'test covers building and reading working copies, but not writing '.
68
+
'to them.'))
69
+
->appendParagraph(
70
+
pht(
71
+
'If you run into write failures despite passing this test, '.
72
+
'it suggests that your setup is nearly correct but authentication '.
73
+
'is probably not fully configured.'))
74
+
->addCancelButton($edit_uri)
75
+
->addSubmitButton(pht('Start Test'));
76
+
}
77
+
78
+
}
+5
src/applications/drydock/controller/DrydockRepositoryOperationViewController.php
+5
src/applications/drydock/controller/DrydockRepositoryOperationViewController.php
···
46
46
->setHeader($header)
47
47
->addPropertyList($properties);
48
48
49
+
$status_view = id(new DrydockRepositoryOperationStatusView())
50
+
->setUser($viewer)
51
+
->setOperation($operation);
52
+
49
53
return $this->buildApplicationPage(
50
54
array(
51
55
$crumbs,
52
56
$object_box,
57
+
$status_view,
53
58
),
54
59
array(
55
60
'title' => $title,
+53
src/applications/drydock/operation/DrydockTestRepositoryOperation.php
+53
src/applications/drydock/operation/DrydockTestRepositoryOperation.php
···
1
+
<?php
2
+
3
+
final class DrydockTestRepositoryOperation
4
+
extends DrydockRepositoryOperationType {
5
+
6
+
const OPCONST = 'test';
7
+
8
+
public function getOperationDescription(
9
+
DrydockRepositoryOperation $operation,
10
+
PhabricatorUser $viewer) {
11
+
return pht('Test Configuration');
12
+
}
13
+
14
+
public function getOperationCurrentStatus(
15
+
DrydockRepositoryOperation $operation,
16
+
PhabricatorUser $viewer) {
17
+
18
+
$repository = $operation->getRepository();
19
+
switch ($operation->getOperationState()) {
20
+
case DrydockRepositoryOperation::STATE_WAIT:
21
+
return pht(
22
+
'Waiting to test configuration for %s...',
23
+
$repository->getMonogram());
24
+
case DrydockRepositoryOperation::STATE_WORK:
25
+
return pht(
26
+
'Testing configuration for %s. This may take a moment if Drydock '.
27
+
'has to clone the repository for the first time.',
28
+
$repository->getMonogram());
29
+
case DrydockRepositoryOperation::STATE_DONE:
30
+
return pht(
31
+
'Success! Automation is configured properly and Drydock can '.
32
+
'operate on %s.',
33
+
$repository->getMonogram());
34
+
}
35
+
}
36
+
37
+
public function applyOperation(
38
+
DrydockRepositoryOperation $operation,
39
+
DrydockInterface $interface) {
40
+
$repository = $operation->getRepository();
41
+
42
+
if ($repository->isGit()) {
43
+
$interface->execx('git status');
44
+
} else if ($repository->isHg()) {
45
+
$interface->execx('hg status');
46
+
} else if ($repository->isSVN()) {
47
+
$interface->execx('svn status');
48
+
} else {
49
+
throw new PhutilMethodNotImplementedException();
50
+
}
51
+
}
52
+
53
+
}
+3
src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php
+3
src/applications/drydock/worker/DrydockRepositoryOperationUpdateWorker.php
+53
src/docs/user/userguide/differential_land.diviner
+53
src/docs/user/userguide/differential_land.diviner
···
1
+
@title Differential User Guide: Automated Landing
2
+
@group userguide
3
+
4
+
Configuring Phabricator so you can "Land Revision" from the web UI.
5
+
6
+
7
+
Overview
8
+
========
9
+
10
+
IMPORTANT: This feature is a prototype and has substantial limitations.
11
+
12
+
Phabricator can be configured so that approved revisions may be published
13
+
directly from the web interface. This can make publishing changes more
14
+
convenient, particularly for open source projects where authors may not have
15
+
commit access to the repository. This document explains the workflow and how to
16
+
configure it.
17
+
18
+
When properly configured, a {nav Land Revision} action will appear in
19
+
Differential. This action works like `arc land` on the command line, and
20
+
merges and publishes the revision.
21
+
22
+
This feature has significant limitations:
23
+
24
+
- This feature is a prototype.
25
+
- This feature is only supported in Git.
26
+
- This feature always lands changes onto `master`.
27
+
- This feature does not currently provide chain of custody, and what lands
28
+
may be arbitrarily different than what is shown in Differential.
29
+
30
+
To be landable, a revision must satisfy these requirements:
31
+
32
+
- It must belong to a repository which is tracked in Diffusion
33
+
(both hosted and imported repositories will work).
34
+
- The repository must have a **Staging Area** configured.
35
+
- The repository must have **Repository Automation** configured. For
36
+
details, see @{article:Drydock User Guide: Repository Automation}.
37
+
- The revision must have been created with `arc diff` and pushed to the
38
+
configured staging area at creation time.
39
+
- The user clicking the "Land Revision" button must have permission to push
40
+
to the repository.
41
+
42
+
If these requirements are met, the {nav Land Revision} action should be
43
+
available in the UI.
44
+
45
+
46
+
Next Steps
47
+
==========
48
+
49
+
Continue by:
50
+
51
+
- configuring repository automation with
52
+
@{article:Drydock User Guide: Repository Automation}; or
53
+
- returning to the @{article:Differential User Guide}.
+197
-6
src/docs/user/userguide/drydock.diviner
+197
-6
src/docs/user/userguide/drydock.diviner
···
15
15
you will configure Drydock to enable capabilities in other applications:
16
16
17
17
- Harbormaster can use Drydock to host builds.
18
-
- In the future, Differential will be able to use Drydock to perform
19
-
server-side merges.
18
+
- Differential can use Drydock to perform server-side merges.
20
19
21
20
Users will not normally interact with Drydock directly.
22
21
22
+
If you want to get started with Drydock right away, see
23
+
@{article:Drydock User Guide: Quick Start} for specific instructions on
24
+
configuring integrations.
25
+
23
26
24
27
What Drydock Does
25
28
=================
26
29
27
-
Drydock manages working copies, build hosts, and other software and hardware
30
+
Drydock manages working copies, hosts, and other software and hardware
28
31
resources that build and deployment processes may require in order to perform
29
32
useful work.
30
33
···
49
52
50
53
Drydock solves these scaling problems by providing a central allocation
51
54
framework for //resources//, which are physical or virtual resources like a
52
-
build host or a working copy. Processes which need to share hardware or
53
-
software can use Drydock to coordinate creation, access, and destruction of
54
-
those resources.
55
+
host or a working copy. Processes which need to share hardware or software can
56
+
use Drydock to coordinate creation, access, and destruction of those resources.
55
57
56
58
Applications ask Drydock for resources matching a description, and it allocates
57
59
a corresponding resource by either finding a suitable unused resource or
58
60
creating a new resource. When work completes, the resource is returned to the
59
61
resource pool or destroyed.
60
62
63
+
64
+
Getting Started with Drydock
65
+
============================
66
+
67
+
In general, you will interact with Drydock by configuring blueprints, which
68
+
tell Drydock how to build resources. You can jump into this topic directly
69
+
in @{article:Drydock Blueprints}.
70
+
71
+
For help on configuring specific application features:
72
+
73
+
- to configure server-side merges from Differential, see
74
+
@{article:Differential User Guide: Automated Landing}.
75
+
76
+
You should also understand the Drydock security model before deploying it
77
+
in a production environment. See @{article:Drydock User Guide: Security}.
78
+
79
+
The remainder of this document has some additional high-level discussion about
80
+
how Drydock works and why it works that way, which may be helpful in
81
+
understanding the application as a whole.
82
+
83
+
84
+
Drydock Concepts
85
+
================
86
+
87
+
The major concepts in Drydock are **Blueprints**, **Resources**, **Leases**,
88
+
and the **Allocator**.
89
+
90
+
**Blueprints** are configuration that tells Drydock how to create resources:
91
+
where it can put them, how to access them, how many it can make at once, who is
92
+
allowed to ask for access to them, how to actually build them, how to clean
93
+
them up when they are no longer in use, and so on.
94
+
95
+
Drydock starts without any blueprints. You'll add blueprints to configure
96
+
Drydock and enable it to satisfy requests for resources. You can learn more
97
+
about blueprints in @{article:Drydock Blueprints}.
98
+
99
+
**Resources** represent things (like hosts or working copies) that Drydock has
100
+
created, is managing the lifecycle for, and can give other applications access
101
+
to.
102
+
103
+
**Leases** are requests for resources with certain qualities by other
104
+
applications. For example, Harbormaster may request a working copy of a
105
+
particular repository so it can run unit tests.
106
+
107
+
The **Allocator** is where Drydock actually does work. It works roughly like
108
+
this:
109
+
110
+
- An application creates a lease describing a resource it needs, and
111
+
uses this lease to ask Drydock for an appropriate resource.
112
+
- Drydock looks at free resources to try to find one it can use to satisfy
113
+
the request. If it finds one, it marks the resource as in use and gives
114
+
the application details about how to access it.
115
+
- If it can't find an appropriate resource that already exists, it looks at
116
+
the blueprints it has configured to try to build one. If it can, it creates
117
+
a new resource, then gives the application access to it.
118
+
- Once the application finishes using the resource, it frees it. Depending
119
+
on configuration, Drydock may reuse it, destroy it, or hold onto it and
120
+
make a decision later.
121
+
122
+
Some minor concepts in Drydock are **Slot Locks** and **Repository Operations**.
123
+
124
+
**Slot Locks** are simple optimistic locks that most Drydock blueprints use to
125
+
avoid race conditions. Their design is not particularly interesting or novel,
126
+
they're just a fairly good fit for most of the locking problems that Drydock
127
+
blueprints tend to encounter and Drydock provides APIs to make them easy to
128
+
work with.
129
+
130
+
**Repository Operations** help other applications coordinate writes to
131
+
repositories. Multiple applications perform similar kinds of writes, and these
132
+
writes require more sequencing/coordination and user feedback than other
133
+
operations.
134
+
135
+
136
+
Architecture Overview
137
+
=====================
138
+
139
+
This section describes some of Drydock's design goals and architectural
140
+
choices, so you can understand its strengths and weaknesses and which problem
141
+
domains it is well or poorly suited for.
142
+
143
+
A typical use case for Drydock is giving another application access to a
144
+
working copy in order to run a build or unit test operation. Drydock can
145
+
satisfy the request and resume execution of application code in 1-2 seconds
146
+
under reasonable conditions and with moderate tradeoffs, and can satisfy a
147
+
large number of these requests in parallel.
148
+
149
+
**Scalable**: Drydock is designed to scale easily to something in the realm of
150
+
thousands of hosts in hundreds of pools, and far beyond that with a little
151
+
work.
152
+
153
+
Drydock is intended to solve resource management problems at very large scales
154
+
and minimzes blocking operations, locks, and artificial sequencing. Drydock is
155
+
designed to fully utilize an almost arbitrarily large pool of resources and
156
+
improve performance roughly linearly with available hardware.
157
+
158
+
Because the application assumes that deployment at this scale and complexity
159
+
level is typical, you may need to configure more things and do more work than
160
+
you would under the simplifying assumptions of small scale.
161
+
162
+
**Heavy Resources**: Drydock assumes that resources are relatively
163
+
heavyweight and and require a meaningful amount (a second or more) of work to
164
+
build, maintain and tear down. It also assumes that leases will often have
165
+
substantial lifespans (seconds or minutes) while performing operations.
166
+
167
+
Resources like working copies (which typically take several seconds to create
168
+
with a command like `git clone`) and VMs (which typically take several seconds
169
+
to spin up) are good fits for Drydock and for the problems it is intended to
170
+
solve.
171
+
172
+
Lease operations like running unit tests, performing builds, executing merges,
173
+
generating documentation and running temporary services (which typically last
174
+
at least a few seconds) are also good fits for Drydock.
175
+
176
+
In both cases, the general concern with lightweight resources and operations is
177
+
that Drydock operation overhead is roughly on the order of a second for many
178
+
tasks, so overhead from Drydock will be substantial if resources are built and
179
+
torn down in a few milliseconds or lease operations require only a fraction of
180
+
a second to execute.
181
+
182
+
As a rule of thumb, Drydock may be a poor fit for a problem if operations
183
+
typically take less than a second to build, execute, and destroy.
184
+
185
+
**Focus on Resource Construction**: Drydock is primarily solving a resource
186
+
construction problem: something needs a resource matching some description, so
187
+
Drydock finds or builds that resource as quickly as possible.
188
+
189
+
Drydock generally prioritizes responding to requests quickly over other
190
+
concerns, like minimizing waste or performing complex scheduling. Although you
191
+
can make adjustments to some of these behaviors, it generally assumes that
192
+
resources are cheap compared to the cost of waiting for resource construction.
193
+
194
+
This isn't to say that Drydock is grossly wasteful or has a terrible scheduler,
195
+
just that efficient utilization and efficient scheduling aren't the primary
196
+
problems the design focuses on.
197
+
198
+
This prioritization corresponds to scenarios where resources are something like
199
+
hosts or working copies, and operations are something like builds, and the cost
200
+
of hosts and storage is small compared to the cost of engineer time spent
201
+
waiting on jobs to get scheduled.
202
+
203
+
Drydock may be a weak fit for a problem if it is bounded by resource
204
+
availability and using resources as efficiently as possible is very important.
205
+
Drydock generally assumes you will respond to a resource deficit by making more
206
+
resources available (usually very cheap), rather than by paying engineers to
207
+
wait for operations to complete (usually very expensive).
208
+
209
+
**Isolation Tradeoffs**: Drydock assumes that multiple operations running at
210
+
similar levels of trust may be interested in reducing isolation to improve
211
+
performance, reduce complexity, or satisfy some other similar goal. It does not
212
+
guarantee isolation and assumes most operations will not run in total isolation.
213
+
214
+
If this isn't true for your use case, you'll need to be careful in configuring
215
+
Drydock to make sure that operations are fully isolated and can not interact.
216
+
Complete isolation will reduce the performance of the allocator as it will
217
+
generally prevent it from reusing resources, which is one of the major ways it
218
+
can improve performance.
219
+
220
+
You can find more discussion of these tradeoffs in
221
+
@{article:Drydock User Guide: Security}.
222
+
223
+
**Agentless**: Drydock does not require an agent or daemon to be installed on
224
+
hosts. It interacts with hosts over SSH.
225
+
226
+
**Very Abstract**: Drydock's design is //extremely// abstract. Resources have
227
+
very little hardcoded behavior. The allocator has essentially zero specialized
228
+
knowledge about what it is actually doing.
229
+
230
+
One aspect of this abstractness is that Drydock is composable, and solves
231
+
complex allocation problems by //asking itself// to build the pieces it needs.
232
+
To build a working copy, Drydock first asks itself for a suitable host. It
233
+
solves this allocation sub-problem, then resolves the original request.
234
+
235
+
This allows new types of resources to build on Drydock's existing knowledge of
236
+
resource construction by just saying "build one of these other things you
237
+
already know how to build, then apply a few adjustments". This also means that
238
+
you can tell Drydock about a new way to build hosts (say, bring up VMs from a
239
+
different service provider) and the rest of the pipeline can use these new
240
+
hosts interchangeably with the old hosts.
241
+
242
+
While this design theoretically makes Drydock more powerful and more flexible
243
+
than a less abstract approach, abstraction is frequently a double-edged sword.
244
+
245
+
Drydock is almost certainly at the extreme upper end of abstraction for tools
246
+
in this space, and the level of abstraction may ultimately match poorly with a
247
+
particular problem domain. Alternative approaches may give you more specialized
248
+
and useful tools for approaching a given problem.
249
+
250
+
61
251
Next Steps
62
252
==========
63
253
···
65
255
66
256
- understanding Drydock security concerns with
67
257
@{article:Drydock User Guide: Security}; or
258
+
- learning about blueprints in @{article:Drydock Blueprints}; or
68
259
- allowing Phabricator to write to repositories with
69
260
@{article:Drydock User Guide: Repository Automation}.
+80
src/docs/user/userguide/drydock_blueprints.diviner
+80
src/docs/user/userguide/drydock_blueprints.diviner
···
1
+
@title Drydock Blueprints
2
+
@group userguide
3
+
4
+
Overview of Drydock blueprint types.
5
+
6
+
7
+
Overview
8
+
========
9
+
10
+
IMPORTANT: Drydock is not a mature application and may be difficult to
11
+
configure and use for now.
12
+
13
+
Drydock builds and manages various hardware and software resources, like
14
+
hosts and repository working copies. Other applications can use these resources
15
+
to perform useful work (like running tests or builds).
16
+
17
+
For additional disussion of Drydock, see @{article:Drydock User Guide}.
18
+
19
+
Drydock can't create any resources until you configure it. You'll configure
20
+
Drydock by creating **Blueprints**. Each blueprint tells Drydock how to build
21
+
a specific kind of resource, how many it is allowed to build, where it should
22
+
build them, who is authorized to request them, and so on.
23
+
24
+
You can create a new blueprint in Drydock from the web UI:
25
+
26
+
{nav Drydock > Blueprints > New Blueprint}
27
+
28
+
Each blueprint builds resources of a specific type, like hosts or repository
29
+
working copies. Detailed topic guides are available for each resource type:
30
+
31
+
**Hosts**: Hosts are the building block for most other resources. For details,
32
+
see @{article:Drydock Blueprints: Hosts}.
33
+
34
+
**Working Copies**: Working copies allow Drydock to perform repository
35
+
operations like running tests, performing builds, and handling merges.
36
+
37
+
38
+
Authorizing Access
39
+
==================
40
+
41
+
Before objects in other applications can use a blueprint, the blueprint must
42
+
authorize them.
43
+
44
+
This mostly serves to prevent users with limited access from executing
45
+
operations on trusted hosts. For additional discussion, see
46
+
@{article:Drydock User Guide: Security}.
47
+
48
+
This also broadly prevents Drydock from surprising you by coming up with a
49
+
valid but unintended solution to an allocation problem which runs some
50
+
operation on resources that are techincally suitable but not desirable. For
51
+
example, you may not want your Android builds running on your iPhone build
52
+
tier, even if there's no technical reason they can't.
53
+
54
+
You can review active authorizations and pending authorization requests in
55
+
the "Active Authorizations" section of the blueprint detail screen.
56
+
57
+
To approve an authorization, click it and select {nav Approve Authorization}.
58
+
Until you do, the requesting object won't be able to access resources from
59
+
the blueprint.
60
+
61
+
You can also decline an authorization. This prevents use of resources and
62
+
removes it from the authorization approval queue.
63
+
64
+
65
+
Disabling Blueprints
66
+
====================
67
+
68
+
You can disable a blueprint by selecting {nav Disable Blueprint} from the
69
+
blueprint detail screen.
70
+
71
+
Disabled blueprints will no longer be used for new allocations. However,
72
+
existing resources will continue to function.
73
+
74
+
75
+
Next Steps
76
+
==========
77
+
78
+
Continue by:
79
+
80
+
- returning to the @{article:Drydock User Guide}.
+126
src/docs/user/userguide/drydock_hosts.diviner
+126
src/docs/user/userguide/drydock_hosts.diviner
···
1
+
@title Drydock Blueprints: Hosts
2
+
@group userguide
3
+
4
+
Guide to configuring Drydock host blueprints.
5
+
6
+
7
+
Overview
8
+
========
9
+
10
+
IMPORTANT: Drydock is not a mature application and may be difficult to
11
+
configure and use for now.
12
+
13
+
To give Drydock access to machines so it can perform work, you'll configure
14
+
**host blueprints**. These blueprints tell Drydock where to find machines (or
15
+
how to build machines) and how to connect to them.
16
+
17
+
Once Drydock has access to hosts it can use them to build more interesting and
18
+
complex types of resources, like repository working copies.
19
+
20
+
Drydock currently supports these kinds of host blueprints:
21
+
22
+
- **Almanac Hosts**: Gives Drydock access to a predefined list of hosts.
23
+
24
+
Drydock may support additional blueprints in the future.
25
+
26
+
27
+
Security
28
+
========
29
+
30
+
Drydock can be used to run semi-trusted and untrusted code, and you may want
31
+
to isolate specific processes or classes of processes from one another. See
32
+
@{article:Drydock User Guide: Security} for discussion of security
33
+
concerns and guidance on how to make isolation tradeoffs.
34
+
35
+
36
+
General Considerations
37
+
======================
38
+
39
+
**You must install software on hosts.** Drydock does not currently handle
40
+
installing software on hosts. You'll need to make sure any hosts are configured
41
+
properly with any software you need, and have tools like `git`, `hg` or `svn`
42
+
that may be required to interact with working copies.
43
+
44
+
You do **not** need to install PHP, arcanist, libphutil or Phabricator on the
45
+
hosts unless you are specifically running `arc` commands.
46
+
47
+
**You must configure authentication.** Drydock also does not handle credentials
48
+
for VCS operations. If you're interacting with repositories hosted on
49
+
Phabricator, the simplest way to set this up is something like this:
50
+
51
+
- Create a new bot user in Phabricator.
52
+
- In {nav Settings > SSH Public Keys}, add a public key or generate a
53
+
keypair.
54
+
- Put the private key on your build hosts as `~/.ssh/id_rsa` for whatever
55
+
user you're connecting with.
56
+
57
+
This will let processes on the host access Phabricator as the bot user, and
58
+
use the bot user's permissions to pull and push changes.
59
+
60
+
If you're using hosted repositories from an external service, you can follow
61
+
similar steps for that service.
62
+
63
+
Note that any processes running under the given user account will have access
64
+
to the private key, so you should give the bot the smallest acceptable level of
65
+
permissions if you're running semi-trusted or untrusted code like unit tests.
66
+
67
+
**You must create a `/var/drydock` directory.** This is hard-coded in Drydock
68
+
for now, so you need to create it on the hosts. This can be a symlink to
69
+
a different location if you prefer.
70
+
71
+
72
+
Almanac Hosts
73
+
=============
74
+
75
+
The **Almanac Hosts** blueprint type gives Drydock access to a predefined list
76
+
of hosts which you configure in the Almanac application. This is the simplest
77
+
type of blueprint to set up.
78
+
79
+
For more information about Almanac, see @{article:Almanac User Guide}.
80
+
81
+
For example, suppose you have `build001.mycompany.com` and
82
+
`build002.mycompany.com`, and want to configure Drydock to be able to use these
83
+
hosts. To do this:
84
+
85
+
**Create Almanac Devices**: Create a device record in Almanac for each your
86
+
hosts.
87
+
88
+
{nav Almanac > Devices > Create Device}
89
+
90
+
Enter the device names (like `build001.mycompany.com`). After creating the
91
+
devices, use {nav Add Interface} to configure the ports and IP addresses that
92
+
Drydock should connect to over SSH (normally, this is port `22`).
93
+
94
+
**Create an Almanac Service**: In the Almanac application, create a new service
95
+
to define the pool of devices you want to use.
96
+
97
+
{nav Almanac > Services > Create Service}
98
+
99
+
Choose the service type **Drydock: Resource Pool**. This will allow Drydock
100
+
to use the devices that are bound to the service.
101
+
102
+
Now, use {nav Add Binding} to bind all of the devices to the service.
103
+
104
+
You can add more hosts to the pool later by binding additional devices, and
105
+
Drydock will automatically start using them. Likewise, you can remove bindings
106
+
to take hosts out of service.
107
+
108
+
**Create a Drydock Blueprint**: Now, create a new blueprint in Drydock.
109
+
110
+
{nav Drydock > Blueprints > New Blueprint}
111
+
112
+
Choose the **Almanac Hosts** blueprint type.
113
+
114
+
In **Almanac Services**, select the service you previously created. For
115
+
**Credentials**, select an SSH private key you want Drydock to use to connect
116
+
to the hosts.
117
+
118
+
Drydock should now be able to build resources from these hosts.
119
+
120
+
121
+
Next Steps
122
+
==========
123
+
124
+
Continue by:
125
+
126
+
- returning to @{article:Drydock Blueprints}.
+74
src/docs/user/userguide/drydock_quick_start.diviner
+74
src/docs/user/userguide/drydock_quick_start.diviner
···
1
+
@title Drydock User Guide: Quick Start
2
+
@group userguide
3
+
4
+
Guide to getting Drydock
5
+
6
+
Quick Start: Land Revisions
7
+
===========================
8
+
9
+
Quick start guide to getting "Land Revision" working in Differential. For
10
+
a more detailed guide, see @{article:Drydock User Guide: Repository Automation}.
11
+
12
+
Choose a repository you want to enable "Land Revision" for. We'll call this
13
+
**Repository X**.
14
+
15
+
You need to configure a staging area for this repository if you haven't
16
+
already. You can do this in Diffusion in {nav Edit Repository > Edit Staging}.
17
+
We'll call this **Staging Area Y**.
18
+
19
+
Choose or create a host you want to run merges on. We'll call this
20
+
`automation001`. For example, you might bring up a new host in EC2 and
21
+
label it `automation001.mycompany.com`. You can use an existing host if you
22
+
prefer.
23
+
24
+
Create a user account on the host, or choose an existing user account. This is
25
+
the user that merges will execute under: Drydock will connect to it and run a
26
+
bunch of `git` commands, then ultimately run `git push`. We'll call this user
27
+
`builder`.
28
+
29
+
Install `git`, `hg` or `svn` if you haven't already and set up private keys
30
+
for `builder` so it can pull and push any repositories you want to operate
31
+
on.
32
+
33
+
If your repository and/or staging area are hosted in Phabricator, you may want
34
+
to create a corresponding bot account so you can add keys and give it
35
+
permissions.
36
+
37
+
At this point you should be able to `ssh builder@automation001` to connect to
38
+
the host, and get a normal shell. You should be able to `git clone ...` from
39
+
**Repository X** and from **Staging Area Y**, and `git push` to **Repository
40
+
X**. If you can't, configure things so you can.
41
+
42
+
Now, create a host blueprint for the host. You can find a more detailed
43
+
walkthrough in @{article:Drydock Blueprints: Hosts}. Briefly:
44
+
45
+
- Create an Almanac device for the host. This should have the IP address and
46
+
port for your host.
47
+
- Create an Almanac service bound to the device. This should be a Drydock
48
+
resource pool service and have a binding to the IP from the previous step.
49
+
- Create a Drydock host blueprint which uses the service from the previous
50
+
step. It should be configured with an SSH private key that can be used
51
+
to connect to `builder@automation001`.
52
+
53
+
Then, create a new working copy blueprint which uses the host blueprint you
54
+
just made. You can find a more detailed walkthrough in @{article:Drydock
55
+
Blueprints: Working Copies}. Authorize the working copy blueprint to use the
56
+
host blueprint.
57
+
58
+
Finally, configure repository automation for **Repository X**:
59
+
{nav Edit Repository > Edit Automation}. Provide the working copy blueprint
60
+
from the previous step. Authorize the repository to use the working copy
61
+
blueprint.
62
+
63
+
After you save changes, click {nav Test Configuration} to test that things
64
+
are working properly.
65
+
66
+
The "Land Revision" action should now be available on revisions for this
67
+
repository.
68
+
69
+
Next Steps
70
+
==========
71
+
72
+
Continue by:
73
+
74
+
- returning to @{article:Drydock User Guide}.
+49
-3
src/docs/user/userguide/drydock_repository_automation.diviner
+49
-3
src/docs/user/userguide/drydock_repository_automation.diviner
···
7
7
Overview
8
8
========
9
9
10
-
IMPORTANT: This feature is very new and most of the capabilities described
10
+
IMPORTANT: This feature is very new and some of the capabilities described
11
11
in this document are not yet available. This feature as a whole is a prototype.
12
12
13
13
By configuring Drydock and Diffusion appropriately, you can enable **Repository
14
-
Automation** for a repository. Once automation is set up, Phabricator will be
15
-
able to make changes to the repository.
14
+
Automation** for a repository. This will allow Phabricator to make changes
15
+
to the repository.
16
+
17
+
18
+
Limitations
19
+
===========
20
+
21
+
- This feature is a prototype.
22
+
- Only Git is supported.
16
23
17
24
18
25
Security
···
27
34
run tests and automation on the same hardware, tests may be able to interfere
28
35
with automation. You can read more about this in
29
36
@{article:Drydock User Guide: Security}.
37
+
38
+
39
+
Configuring Automation
40
+
======================
41
+
42
+
To configure automation, use {nav Edit Repository > Edit Automation} from
43
+
Diffusion.
44
+
45
+
On the configuration screen, specify one or more working copy blueprints in
46
+
Drydock (usually, you'll just use one). Repository automation will use working
47
+
copies built by these blueprints to perform merges and push changes.
48
+
49
+
For more details on configuring these blueprints, see
50
+
@{article:Drydock Blueprints: Working Copies}.
51
+
52
+
After selecting one or more blueprints, make sure you authorize the repository
53
+
to use them. Automation operations won't be able to proceed until you do. The
54
+
UI will remind you if you have unauthorized blueprints selected.
55
+
56
+
57
+
Testing Configuration
58
+
=====================
59
+
60
+
Once the blueprints are configured and authorized, use {nav Test Configuration}
61
+
to check that things are configured correctly. This will build a working copy
62
+
in Drydock, connect to it, and run a trivial command (like `git status`) to
63
+
make sure things work.
64
+
65
+
If it's the first time you're doing this, it may take a few moments since it
66
+
will need to clone a fresh working copy.
67
+
68
+
If the test is successful, your configuration is generally in good shape. If
69
+
not, it should give you more details about what went wrong.
70
+
71
+
Since the test doesn't actually do a push, it's possible that you may have
72
+
everything configured properly //except// write access. In this case, you'll
73
+
run into a permission error when you try to actually perform a merge or other
74
+
similar write. If you do, adjust permissions or credentials appropriately so
75
+
the working copy can be pushed from.
30
76
31
77
32
78
Next Steps
+39
src/docs/user/userguide/drydock_working_copies.diviner
+39
src/docs/user/userguide/drydock_working_copies.diviner
···
1
+
@title Drydock Blueprints: Working Copies
2
+
@group userguide
3
+
4
+
Guide to configuring Drydock working copy blueprints.
5
+
6
+
7
+
Overview
8
+
========
9
+
10
+
IMPORTANT: Drydock is not a mature application and may be difficult to
11
+
configure and use for now.
12
+
13
+
To let Drydock build repository working copies in order to run unit tests and
14
+
other similar operations, you'll configure **working copy blueprints**.
15
+
16
+
Working Copies
17
+
==============
18
+
19
+
Working copy blueprints rely on host blueprints, so you'll need to configure
20
+
a suitable host blueprint first. See @{article:Drydock Blueprints: Hosts}.
21
+
22
+
To configure a working copy blueprint, choose the host blueprints it should
23
+
use in **Use Blueprints**.
24
+
25
+
You can optionally specify a **Limit**. If you do, the blueprint won't be
26
+
allowed to create more than this many simultaneous resources. If you leave
27
+
it empty, the blueprint will be able to create an unlimited number of
28
+
resources.
29
+
30
+
After you save the blueprint, make sure you authorize it to use the selected
31
+
host blueprints. It won't be able to acquire host resources until you do.
32
+
33
+
34
+
Next Steps
35
+
==========
36
+
37
+
Continue by:
38
+
39
+
- returning to @{article:Drydock Blueprints}.