this repo has no description
internal/ci: further tidy up
* Have our caches work in read-write mode when running on the test
default branch (generally ci/test). This allows us to more easily test
whether a cache-related change will work, with repeated pushes/runs to
the change on the ci/test branch.
* Make dummy dispatch data optional for the trybot_dispatch workflow
when pushed to ci/test. We correspondingly extend the GitHub
expression that determines when to run the dispatch workflow to
include a check to see whether the dummy data is valid or not.
* Declare isTestDefaultBranch as a reusable means of checking in an 'if'
condition whether we are running on the test default branch.
* Declare a field that captures the GitHub expression that can be used
to dereference the decoded Dispatch-Trailer value.
* Do not run evict_caches on pushes to the default test branch. Evict
caches deletes all existing caches on the main and trybot repos and
as such should only be run with care, when testing changes to this
workflow, and when nobody else is relying on the caches! Otherwise
it's too easy to forget that pushing to ci/test (which should be safe)
can cause the trybot runs to blow up in terms of runtimes.
* Extend the existing setupGoActionsCaches template with a parameter
#additionalCacheDirs that allows additional paths to be to the caching
step. At this point, the template "setupGoActionsCaches" is poorly
named because there isn't anything unique to Go happening here. Just
that we need to do more lifting when it comes to Go caches than
anything else.
* Remove internal/ci/github/gen_trybot_dispatch.cue to remove the fake
dispatch data.
Signed-off-by: Paul Jolly <paul@myitcv.io>
Change-Id: Ifd454d072b61c4ffb080a870fdf7638e10bd4590
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/552365
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
TryBot-Result: CUEcueckoo <cueckoo+gerrithub@cuelang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
+44
-32
-3
.github/workflows/evict_caches.yml
-3
.github/workflows/evict_caches.yml
+4
-4
.github/workflows/trybot.yml
+4
-4
.github/workflows/trybot.yml
···
87
87
name: Get go build/test cache directory
88
88
run: echo "dir=$(go env GOCACHE)" >> ${GITHUB_OUTPUT}
89
89
- if: |-
90
-
((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
91
-
Dispatch-Trailer: {"type":"'))))
90
+
(((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
91
+
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
92
92
uses: actions/cache@v3
93
93
with:
94
94
path: |-
···
97
97
key: ${{ runner.os }}-${{ matrix.go-version }}-${{ github.run_id }}
98
98
restore-keys: ${{ runner.os }}-${{ matrix.go-version }}
99
99
- if: |-
100
-
! ((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
101
-
Dispatch-Trailer: {"type":"'))))
100
+
! (((github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-branch.')) && (! (contains(github.event.head_commit.message, '
101
+
Dispatch-Trailer: {"type":"')))) || (github.ref == 'refs/heads/ci/test'))
102
102
uses: actions/cache/restore@v3
103
103
with:
104
104
path: |-
+3
-3
.github/workflows/trybot_dispatch.yml
+3
-3
.github/workflows/trybot_dispatch.yml
···
12
12
defaults:
13
13
run:
14
14
shell: bash
15
-
if: ${{ github.ref == 'refs/heads/ci/test' || github.event.client_payload.type == 'trybot' }}
15
+
if: ${{ ((github.ref == 'refs/heads/ci/test') && false) || github.event.client_payload.type == 'trybot' }}
16
16
steps:
17
17
- name: Write netrc file for cueckoo Gerrithub
18
18
run: |-
···
23
23
EOD
24
24
chmod 600 ~/.netrc
25
25
- id: payload
26
-
if: github.repository == 'cue-lang/cue' && github.ref == 'refs/heads/ci/test'
26
+
if: github.repository == 'cue-lang/cue' && (github.ref == 'refs/heads/ci/test')
27
27
name: Write fake payload
28
28
run: |-
29
29
cat <<EOD >> $GITHUB_OUTPUT
30
30
value<<DOE
31
-
{"type":"trybot","CL":552503,"patchset":2,"targetBranch":"master","ref":"refs/changes/03/552503/2"}
31
+
null
32
32
DOE
33
33
EOD
34
34
- if: github.event.client_payload.type != 'trybot'
+7
-9
internal/ci/base/gerrithub.cue
+7
-9
internal/ci/base/gerrithub.cue
···
31
31
}
32
32
33
33
trybotDispatchWorkflow: bashWorkflow & {
34
-
#dummyDispatch: #dispatch
35
-
name: "Dispatch \(trybot.key)"
34
+
#dummyDispatch?: #dispatch
35
+
name: "Dispatch \(trybot.key)"
36
36
on: {
37
37
repository_dispatch: {}
38
38
push: {
···
44
44
jobs: {
45
45
(trybot.key): {
46
46
"runs-on": linuxMachine
47
+
48
+
let goodDummyData = [ if encjson.Marshal(#dummyDispatch) != _|_ {true}, false][0]
47
49
48
50
// We set the "on" conditions above, but this would otherwise mean we
49
51
// run for all dispatch events.
50
-
if: "${{ github.ref == 'refs/heads/\(testDefaultBranch)' || github.event.client_payload.type == '\(trybot.key)' }}"
52
+
if: "${{ (\(isTestDefaultBranch) && \(goodDummyData)) || github.event.client_payload.type == '\(trybot.key)' }}"
51
53
52
54
// See the comment below about the need for cases
53
55
let cases = [
···
69
71
json.#step & {
70
72
name: "Write fake payload"
71
73
id: "payload"
72
-
if: "github.repository == '\(githubRepositoryPath)' && github.ref == 'refs/heads/\(testDefaultBranch)'"
74
+
if: "github.repository == '\(githubRepositoryPath)' && \(isTestDefaultBranch)"
73
75
74
76
// Use bash heredocs so that JSON's use of double quotes does
75
77
// not get interpreted as shell. Both in the running of the
···
78
80
run: #"""
79
81
cat <<EOD >> $GITHUB_OUTPUT
80
82
value<<DOE
81
-
\#(encjson.Marshal(#dummyDispatch))
83
+
\#(*encjson.Marshal(#dummyDispatch) | "null")
82
84
DOE
83
85
EOD
84
86
"""#
···
236
238
schedule: [
237
239
{cron: "0 2 * * *"},
238
240
]
239
-
push: {
240
-
// To enable testing of the dispatch itself
241
-
branches: [testDefaultBranch]
242
-
}
243
241
}
244
242
245
243
jobs: {
+30
-7
internal/ci/base/github.cue
+30
-7
internal/ci/base/github.cue
···
4
4
5
5
import (
6
6
encjson "encoding/json"
7
+
"list"
7
8
"strings"
8
9
"strconv"
9
10
···
62
63
},
63
64
64
65
{
65
-
let stepName = strings.Replace(dispatchTrailer, "-", "", -1)
66
66
json.#step & {
67
67
name: "Try to extract \(dispatchTrailer)"
68
-
id: stepName
68
+
id: dispatchTrailerStepID
69
69
run: """
70
70
x="$(git log -1 --pretty='%(trailers:key=\(dispatchTrailer),valueonly)')"
71
71
if [[ "$x" == "" ]]
···
78
78
# output from this step is the JSON value null or not.
79
79
x=null
80
80
fi
81
-
echo "value<<EOD" >> $GITHUB_OUTPUT
81
+
echo "\(_dispatchTrailerDecodeStepOutputVar)<<EOD" >> $GITHUB_OUTPUT
82
82
echo "$x" >> $GITHUB_OUTPUT
83
83
echo "EOD" >> $GITHUB_OUTPUT
84
84
"""
···
180
180
#readonly: *false | bool
181
181
#cleanTestCache: *!#readonly | bool
182
182
#goVersion: string
183
-
#os: string
183
+
#additionalCacheDirs: [...string]
184
+
#os: string
184
185
185
186
let goModCacheDirID = "go-mod-cache-dir"
186
187
let goCacheDirID = "go-cache-dir"
···
188
189
// cacheDirs is a convenience variable that includes
189
190
// GitHub expressions that represent the directories
190
191
// that participate in Go caching.
191
-
let cacheDirs = [ "${{ steps.\(goModCacheDirID).outputs.dir }}/cache/download", "${{ steps.\(goCacheDirID).outputs.dir }}"]
192
+
let cacheDirs = list.Concat([[
193
+
"${{ steps.\(goModCacheDirID).outputs.dir }}/cache/download",
194
+
"${{ steps.\(goCacheDirID).outputs.dir }}",
195
+
], #additionalCacheDirs])
192
196
193
197
let cacheRestoreKeys = "\(#os)-\(#goVersion)"
194
198
···
205
209
}
206
210
}
207
211
212
+
let readWriteCacheExpr = "(\(isProtectedBranch) || \(isTestDefaultBranch))"
213
+
208
214
// pre is the list of steps required to establish and initialise the correct
209
215
// caches for Go-based workflows.
210
216
[
···
227
233
// it's impossible for anything else to write such a cache.
228
234
if !#readonly {
229
235
cacheStep & {
230
-
if: isProtectedBranch
236
+
if: readWriteCacheExpr
231
237
uses: "actions/cache@v3"
232
238
}
233
239
},
···
238
244
// are not readonly, then we need to predicate this step on us not
239
245
// being on a protected branch.
240
246
if !#readonly {
241
-
if: "! \(isProtectedBranch)"
247
+
if: "! \(readWriteCacheExpr)"
242
248
}
243
249
244
250
uses: "actions/cache/restore@v3"
···
271
277
(_matchPattern & {variable: "github.ref", pattern: "refs/heads/\(branch)"}).expr
272
278
}], " || ") + ") && (! \(containsDispatchTrailer)))"
273
279
}
280
+
281
+
// isTestDefaultBranch is an expression that evaluates to true if
282
+
// the job is running on the testDefaultBranch
283
+
isTestDefaultBranch: "(github.ref == 'refs/heads/\(testDefaultBranch)')"
274
284
275
285
// #isReleaseTag creates a GitHub expression, based on the given release tag
276
286
// pattern, that evaluates to true if called in the context of a workflow that
···
301
311
//
302
312
// NOTE: keep this consistent with gerritstatusupdater parsing logic.
303
313
dispatchTrailer: "Dispatch-Trailer"
314
+
315
+
// dispatchTrailerStepID is the ID of the step that attempts
316
+
// to extract a Dispatch-Trailer value from the commit at HEAD
317
+
dispatchTrailerStepID: strings.Replace(dispatchTrailer, "-", "", -1)
318
+
319
+
// _dispatchTrailerDecodeStepOutputVar is the name of the output
320
+
// variable int he dispatchTrailerStepID step
321
+
_dispatchTrailerDecodeStepOutputVar: "value"
322
+
323
+
// dispatchTrailerExpr is a GitHub expression that can be dereferenced
324
+
// to get values from the JSON-decded Dispatch-Trailer value that
325
+
// is extracted during the dispatchTrailerStepID step.
326
+
dispatchTrailerExpr: "fromJSON(steps.\(dispatchTrailerStepID).outputs.\(_dispatchTrailerDecodeStepOutputVar))"
304
327
305
328
// containsDispatchTrailer returns a GitHub expression that looks at the commit
306
329
// message of the head commit of the event that triggered the workflow, an