this repo has no description
internal/ci: split Linux runners into "small" and "large"
Cheap workflows like "tip triggers" and "trybot dispatch" do not need
as much CPU and memory as expensive trybot workflows, so those always
use the small runner.
The trybot runner still uses "linuxMachine", which now defaults to the
small runner, but can be opted into the large runner for those
repositories which really need it. The majority of them should be
perfectly fine on the smaller machine with half as much CPU and memory.
The "small" size has been chosen in a relatively conservative way,
only being half as large as the "large" machine.
This is because even the public GitHub runners have four CPU cores
and 16 GiB of RAM, so going all the way down to 2 CPU cores and 4 GiB
might be too low.
Thankfully, now that the split is made here, and given that these
runner profiles have generic "small" and "large" names, we can later
resize either of them without having to re-configure all repos.
Signed-off-by: Daniel Martí <mvdan@mvdan.cc>
Change-Id: Ib62b2ef79cd7a6d64413b426843abbfdb5627e2a
Reviewed-on: https://cue.gerrithub.io/c/cue-lang/cue/+/1218980
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
Reviewed-by: Paul Jolly <paul@myitcv.io>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
+43
-20
+1
-1
.github/workflows/push_tip_to_trybot.yaml
+1
-1
.github/workflows/push_tip_to_trybot.yaml
+1
-1
.github/workflows/release.yaml
+1
-1
.github/workflows/release.yaml
+1
-1
.github/workflows/tip_triggers.yaml
+1
-1
.github/workflows/tip_triggers.yaml
+12
-12
.github/workflows/trybot.yaml
+12
-12
.github/workflows/trybot.yaml
···
25
25
- 1.23.x
26
26
- 1.24.x
27
27
runner:
28
-
- ns-linux-amd64
28
+
- ns-linux-amd64-large
29
29
- ns-macos-arm64
30
30
- ns-windows-amd64
31
31
runs-on: ${{ matrix.runner }}
···
94
94
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
95
95
run: go env -w GOFLAGS=-count=1
96
96
- run: go run cuelang.org/go/cmd/cue login --token=${{ secrets.NOTCUECKOO_CUE_TOKEN }}
97
-
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
97
+
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
98
98
name: Early git and code sanity checks
99
99
run: go run ./internal/ci/checks
100
100
- if: |-
101
101
((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
102
-
Dispatch-Trailer: {"type":"')))) || !(matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
102
+
Dispatch-Trailer: {"type":"')))) || !(matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
103
103
name: Test
104
104
run: go test ./...
105
-
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
105
+
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
106
106
name: Test with -race
107
107
env:
108
108
GORACE: atexit_sleep_ms=10
109
109
run: go test -race ./...
110
110
- if: |-
111
111
(((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
112
-
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
112
+
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
113
113
name: Test on 32 bits
114
114
env:
115
115
GOARCH: "386"
···
119
119
- id: auth
120
120
if: |-
121
121
github.repository == 'cue-lang/cue' && (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
122
-
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
122
+
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
123
123
name: gcloud auth for end-to-end tests
124
124
uses: google-github-actions/auth@v2
125
125
with:
126
126
credentials_json: ${{ secrets.E2E_GCLOUD_KEY }}
127
127
- if: |-
128
128
github.repository == 'cue-lang/cue' && (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
129
-
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
129
+
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
130
130
name: gcloud setup for end-to-end tests
131
131
uses: google-github-actions/setup-gcloud@v2
132
132
- if: |-
133
133
github.repository == 'cue-lang/cue' && (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
134
-
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
134
+
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
135
135
name: End-to-end test
136
136
env:
137
137
CUE_TEST_TOKEN: ${{ secrets.E2E_PORCUEPINE_CUE_TOKEN }}
138
138
run: |-
139
139
cd internal/_e2e
140
140
go test -race
141
-
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
141
+
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
142
142
name: Go checks
143
143
run: |-
144
144
go vet ./...
145
145
go mod tidy
146
146
(cd internal/_e2e && go test -run=-)
147
-
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
147
+
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
148
148
name: staticcheck
149
149
uses: dominikh/staticcheck-action@v1
150
150
with:
···
153
153
use-cache: false
154
154
- if: |-
155
155
(((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
156
-
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
156
+
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test')) && (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
157
157
name: Check all git tags are available
158
158
run: |-
159
159
cd $(mktemp -d)
···
169
169
echo "Did you forget about refs/attic branches? https://github.com/cue-lang/cue/wiki/Notes-for-project-maintainers"
170
170
exit 1
171
171
fi
172
-
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64')
172
+
- if: (matrix.go-version == '1.24.x' && matrix.runner == 'ns-linux-amd64-large')
173
173
name: Generate
174
174
run: go generate ./...
175
175
- if: always()
+1
-1
.github/workflows/trybot_dispatch.yaml
+1
-1
.github/workflows/trybot_dispatch.yaml
···
11
11
defaults:
12
12
run:
13
13
shell: bash --noprofile --norc -euo pipefail {0}
14
-
runs-on: ns-linux-amd64
14
+
runs-on: ns-linux-amd64-small
15
15
if: ${{ ((github.ref == 'refs/heads/ci/test') && false) || github.event.client_payload.type == 'trybot' }}
16
16
steps:
17
17
- name: Write netrc file for cueckoo Gerrithub
+21
-1
internal/ci/base/base.cue
+21
-1
internal/ci/base/base.cue
···
62
62
63
63
workflowFileExtension: ".yaml"
64
64
65
-
linuxMachine: string | *"ns-linux-amd64"
65
+
// Linux is used by lots of repositories for many workflows,
66
+
// so each repository can decide which jobs really need larger machines.
67
+
// Note that using a larger machine with more CPUs and memory lowers
68
+
// the amount of jobs that can be run concurrently at once.
69
+
//
70
+
// At the time of writing, "small" is 4 CPUs and 8GiB of memory
71
+
// and "large" is 8 CPUs and 16GiB of memory.
72
+
// With our concurrency limit for Linux at 64 CPUs and 128 GiB,
73
+
// using "small" rather than "large" allows 16 rather than 8 jobs at once.
74
+
linuxSmallMachine: "ns-linux-amd64-small"
75
+
linuxLargeMachine: "ns-linux-amd64-large"
76
+
77
+
// By default, the main "trybot" test job is run on the small machine.
78
+
// Note that cheap workflows, or those which don't keep a human waiting,
79
+
// should always use linuxSmallMachine.
80
+
linuxMachine: string | *linuxSmallMachine
81
+
82
+
// MacOS on Namespace doesn't really provide small machines,
83
+
// as the smallest is 6 CPUs and 14 GiB on M2 hardware.
84
+
// Most repos only test on Linux - it's just the cue repo now -
85
+
// so it's actually fine if we always use "large" MacOS and Windows machines.
66
86
macosMachine: string | *"ns-macos-arm64"
67
87
windowsMachine: string | *"ns-windows-amd64"
68
88
+2
-2
internal/ci/base/gerrithub.cue
+2
-2
internal/ci/base/gerrithub.cue
···
47
47
}
48
48
jobs: {
49
49
(trybot.key): {
50
-
"runs-on": linuxMachine
50
+
"runs-on": linuxSmallMachine
51
51
52
52
let goodDummyData = [if json.Marshal(#dummyDispatch) != _|_ {true}, false][0]
53
53
···
165
165
push: branches: protectedBranchPatterns
166
166
}
167
167
jobs: push: {
168
-
"runs-on": linuxMachine
168
+
"runs-on": linuxSmallMachine
169
169
if: "${{github.repository == '\(githubRepositoryPath)'}}"
170
170
}
171
171
+1
-1
internal/ci/github/tip_triggers.cue
+1
-1
internal/ci/github/tip_triggers.cue
···
21
21
name: "Triggers on push to tip"
22
22
on: push: branches: [_repo.defaultBranch, _repo.testDefaultBranch]
23
23
jobs: push: {
24
-
"runs-on": _repo.linuxMachine
24
+
"runs-on": _repo.linuxSmallMachine
25
25
if: "${{github.repository == '\(_repo.githubRepositoryPath)'}}"
26
26
steps: [
27
27
_repo.repositoryDispatch & {
+3
internal/ci/repo/repo.cue
+3
internal/ci/repo/repo.cue
···
26
26
botGitHubUser: "cueckoo"
27
27
botGitHubUserEmail: "cueckoo@gmail.com"
28
28
29
+
// The tests on this repository are pretty CPU-intensive.
30
+
linuxMachine: base.linuxLargeMachine
31
+
29
32
// Use the latest Go version for extra checks,
30
33
// such as running tests with the data race detector.
31
34
// This may be a release candidate if we are late into a Go release cycle.