loading up the forgejo repo on tangled to test page performance
Merge pull request 'ci/tests(e2e): always run e2e tests, but only on changed files' (#5450) from fnetx/e2e-on-changes into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5450
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
+363
-52
+1
-1
.forgejo/workflows-composite/build-backend/action.yaml
+1
-1
.forgejo/workflows-composite/build-backend/action.yaml
+2
-1
.forgejo/workflows-composite/setup-env/action.yaml
+2
-1
.forgejo/workflows-composite/setup-env/action.yaml
···
4
4
- name: setup user and permissions
5
5
run: |
6
6
git config --add safe.directory '*'
7
-
adduser --quiet --comment forgejo --disabled-password forgejo
7
+
# ignore if the user already exists (like with the playwright image)
8
+
adduser --quiet --comment forgejo --disabled-password forgejo || true
8
9
chown -R forgejo:forgejo .
9
10
- uses: https://codeberg.org/fnetx/setup-cache-go@b2214eaf6fb44c7e8512c0f462a2c3ec31f86a73
10
11
with:
-37
.forgejo/workflows/e2e.yml
-37
.forgejo/workflows/e2e.yml
···
1
-
name: e2e
2
-
3
-
on:
4
-
pull_request:
5
-
paths:
6
-
- Makefile
7
-
- playwright.config.js
8
-
- .forgejo/workflows/e2e.yml
9
-
- tests/e2e/**
10
-
- web_src/js/**
11
-
- web_src/css/form.css
12
-
- templates/webhook/shared-settings.tmpl
13
-
- templates/org/team/new.tmpl
14
-
15
-
jobs:
16
-
test-e2e:
17
-
if: ${{ !startsWith(vars.ROLE, 'forgejo-') }}
18
-
runs-on: docker
19
-
container:
20
-
image: 'code.forgejo.org/oci/playwright:latest'
21
-
steps:
22
-
- uses: https://code.forgejo.org/actions/checkout@v4
23
-
- uses: https://code.forgejo.org/actions/setup-go@v5
24
-
with:
25
-
go-version-file: "go.mod"
26
-
- run: |
27
-
git config --add safe.directory '*'
28
-
chown -R forgejo:forgejo .
29
-
- run: |
30
-
su forgejo -c 'make deps-frontend frontend deps-backend'
31
-
- run: |
32
-
su forgejo -c 'make backend'
33
-
- run: |
34
-
su forgejo -c 'make generate test-e2e-sqlite'
35
-
timeout-minutes: 40
36
-
env:
37
-
USE_REPO_TEST_DIR: 1
+45
-1
.forgejo/workflows/testing.yml
+45
-1
.forgejo/workflows/testing.yml
···
36
36
- run: make checks-frontend
37
37
- run: make test-frontend-coverage
38
38
- run: make frontend
39
+
- name: Install zstd for cache saving
40
+
# works around https://github.com/actions/cache/issues/1169, because the
41
+
# consuming job has zstd and doesn't restore the cache otherwise
42
+
run: |
43
+
apt-get update -qq
44
+
apt-get -q install -qq -y zstd
45
+
- name: "Cache frontend build for playwright testing"
46
+
uses: actions/cache/save@v4
47
+
with:
48
+
path: ${{github.workspace}}/public/assets
49
+
key: frontend-build-${{ github.sha }}
39
50
test-unit:
40
51
if: ${{ !startsWith(vars.ROLE, 'forgejo-') }}
41
52
runs-on: docker
···
75
86
RACE_ENABLED: 'true'
76
87
TAGS: bindata
77
88
TEST_ELASTICSEARCH_URL: http://elasticsearch:9200
78
-
test-remote-cacher:
89
+
test-e2e:
79
90
if: ${{ !startsWith(vars.ROLE, 'forgejo-') }}
80
91
runs-on: docker
81
92
needs: [backend-checks, frontend-checks]
93
+
container:
94
+
image: 'code.forgejo.org/oci/playwright:latest'
95
+
steps:
96
+
- uses: https://code.forgejo.org/actions/checkout@v4
97
+
with:
98
+
fetch-depth: 20
99
+
- uses: ./.forgejo/workflows-composite/setup-env
100
+
- name: "Restore frontend build"
101
+
uses: actions/cache/restore@v4
102
+
id: cache-frontend
103
+
with:
104
+
path: ${{github.workspace}}/public/assets
105
+
key: frontend-build-${{ github.sha }}
106
+
- name: "Build frontend (if not cached)"
107
+
if: steps.cache-frontend.outputs.cache-hit != 'true'
108
+
run: |
109
+
su forgejo -c 'make deps-frontend frontend'
110
+
- uses: ./.forgejo/workflows-composite/build-backend
111
+
- name: Get changed files
112
+
id: changed-files
113
+
uses: https://code.forgejo.org/fossdd/changed-files@v45
114
+
with:
115
+
separator: '\n'
116
+
- run: |
117
+
su forgejo -c 'make generate test-e2e-sqlite'
118
+
timeout-minutes: 40
119
+
env:
120
+
USE_REPO_TEST_DIR: 1
121
+
CHANGED_FILES: ${{steps.changed-files.outputs.all_changed_files}}
122
+
test-remote-cacher:
123
+
if: ${{ !startsWith(vars.ROLE, 'forgejo-') }}
124
+
runs-on: docker
125
+
needs: [backend-checks, frontend-checks, test-unit]
82
126
container:
83
127
image: 'code.forgejo.org/oci/node:20-bookworm'
84
128
strategy:
+3
CODEOWNERS
+3
CODEOWNERS
+52
-5
tests/e2e/README.md
+52
-5
tests/e2e/README.md
···
110
110
you can use it to run the test jobs:
111
111
112
112
```
113
-
forgejo-runner exec -W .forgejo/workflows/e2e.yml --event=pull_request
113
+
forgejo-runner exec -W .forgejo/workflows/testing.yml -j test-e2e
114
114
```
115
+
116
+
Note that the CI workflow has some logic to run tests based on changed files only.
117
+
This might conflict with your local setup and not run all the desired tests
118
+
because it might only look at file changes in your latest commit.
115
119
116
120
### Run e2e tests with another database
117
121
···
212
216
213
217
### List related files coverage
214
218
215
-
If you think your playwright tests covers an important aspect of some template, CSS or backend files,
216
-
consider adding the paths to `.forgejo/workflows/e2e.yml` in the path filter.
219
+
To speed up the CI pipelines and avoid running expensive tests too often,
220
+
only a selection of tests is run by default,
221
+
based on the changed files.
222
+
223
+
At the top of each playwright test file,
224
+
list the files or file patterns that are covered by your test.
225
+
Often, these are files that you modified for your feature or bugfix,
226
+
or that you looked at (and might still have open in your IDE),
227
+
because your fix depends on their behaviour.
228
+
229
+
#### Which files to watch?
230
+
231
+
The set of files your test "watches" depends on the kind of test you write.
232
+
If you only test for the presence of an element and do no accessibility or placement checks,
233
+
you won't detect broken visual appearance and there is little reason to watch CSS files.
234
+
235
+
However, if your test also checks that an element is correctly positioned
236
+
(e.g. that it does not overflow the page),
237
+
or has accessibiltiy properties (includes colour contrast),
238
+
also list stylesheets that define the behaviour your test depends on.
239
+
240
+
Watching the place that generate the selectors you use
241
+
(typically templates, but can also be JavaScript)
242
+
is a must, to ensure that someone modifying the markup notices that your selectors fail
243
+
(e.g. because an id or class was renamed).
244
+
245
+
If you are unsure about the exact set of files,
246
+
feel free to ask other contributors.
247
+
248
+
#### How to specify the patterns?
249
+
250
+
You put filenames and patterns as blocks between two `// @watch` comments.
251
+
An example that watches changes on (in order)
252
+
a single file,
253
+
a full recursive subfolder,
254
+
two files with a shorthand pattern,
255
+
and a set of files with a certain ending:
217
256
218
-
It ensures that future modifications to this file will be tested as well.
257
+
~~~
258
+
// @watch start
259
+
// templates/webhook/shared-settings.tmpl
260
+
// templates/repo/settings/**
261
+
// web_src/css/{form,repo}.css
262
+
// web_src/css/modules/*.css
263
+
// @watch end
264
+
~~~
219
265
220
-
Currently, we do not run the e2e tests on all changes.
266
+
The patterns are evaluated on a "first-match" basis.
267
+
Under the hood, [gobwas/glob](https://github.com/gobwas/glob) is used.
+12
tests/e2e/actions.test.e2e.js
+12
tests/e2e/actions.test.e2e.js
···
1
1
// @ts-check
2
+
3
+
// @watch start
4
+
// templates/repo/actions/**
5
+
// web_src/css/actions.css
6
+
// web_src/js/components/ActionRunStatus.vue
7
+
// web_src/js/components/RepoActionView.vue
8
+
// modules/actions/**
9
+
// modules/structs/workflow.go
10
+
// routers/api/v1/repo/action.go
11
+
// routers/web/repo/actions/**
12
+
// @watch end
13
+
2
14
import {expect} from '@playwright/test';
3
15
import {test, login_user, load_logged_in_context} from './utils_e2e.js';
4
16
+114
tests/e2e/changes.go
+114
tests/e2e/changes.go
···
1
+
// Copyright 2024 The Forgejo Authors. All rights reserved.
2
+
// SPDX-License-Identifier: GPL-3.0-or-later
3
+
4
+
package e2e
5
+
6
+
import (
7
+
"bufio"
8
+
"os"
9
+
"strings"
10
+
11
+
"code.gitea.io/gitea/modules/log"
12
+
13
+
"github.com/gobwas/glob"
14
+
)
15
+
16
+
var (
17
+
changesetFiles []string
18
+
changesetAvailable bool
19
+
globalFullRun bool
20
+
)
21
+
22
+
func initChangedFiles() {
23
+
var changes string
24
+
changes, changesetAvailable = os.LookupEnv("CHANGED_FILES")
25
+
// the output of the Action seems to actually contain \n and not a newline literal
26
+
changesetFiles = strings.Split(changes, `\n`)
27
+
log.Info("Only running tests covered by a subset of test files. Received the following list of CHANGED_FILES: %q", changesetFiles)
28
+
29
+
globalPatterns := []string{
30
+
// meta and config
31
+
"Makefile",
32
+
"playwright.config.js",
33
+
".forgejo/workflows/testing.yml",
34
+
"tests/e2e/*.go",
35
+
"tests/e2e/shared/*",
36
+
// frontend files
37
+
"frontend/*.js",
38
+
"frontend/{base,index}.css",
39
+
// templates
40
+
"templates/base/**",
41
+
}
42
+
fullRunPatterns := []glob.Glob{}
43
+
for _, expr := range globalPatterns {
44
+
fullRunPatterns = append(fullRunPatterns, glob.MustCompile(expr, '.', '/'))
45
+
}
46
+
globalFullRun = false
47
+
for _, changedFile := range changesetFiles {
48
+
for _, pattern := range fullRunPatterns {
49
+
if pattern.Match(changedFile) {
50
+
globalFullRun = true
51
+
log.Info("Changed files match global test pattern, running all tests")
52
+
return
53
+
}
54
+
}
55
+
}
56
+
}
57
+
58
+
func canSkipTest(testFile string) bool {
59
+
// run all tests when environment variable is not set or changes match global pattern
60
+
if !changesetAvailable || globalFullRun {
61
+
return false
62
+
}
63
+
64
+
for _, changedFile := range changesetFiles {
65
+
if strings.HasSuffix(testFile, changedFile) {
66
+
return false
67
+
}
68
+
for _, pattern := range getWatchPatterns(testFile) {
69
+
if pattern.Match(changedFile) {
70
+
return false
71
+
}
72
+
}
73
+
}
74
+
return true
75
+
}
76
+
77
+
func getWatchPatterns(filename string) []glob.Glob {
78
+
file, err := os.Open(filename)
79
+
if err != nil {
80
+
log.Fatal(err.Error())
81
+
}
82
+
defer file.Close()
83
+
scanner := bufio.NewScanner(file)
84
+
85
+
watchSection := false
86
+
patterns := []glob.Glob{}
87
+
for scanner.Scan() {
88
+
line := scanner.Text()
89
+
// check for watch block
90
+
if strings.HasPrefix(line, "// @watch") {
91
+
if watchSection {
92
+
break
93
+
}
94
+
watchSection = true
95
+
}
96
+
if !watchSection {
97
+
continue
98
+
}
99
+
100
+
line = strings.TrimPrefix(line, "// ")
101
+
if line != "" {
102
+
globPattern, err := glob.Compile(line, '.', '/')
103
+
if err != nil {
104
+
log.Fatal("Invalid glob pattern '%s' (skipped): %v", line, err)
105
+
}
106
+
patterns = append(patterns, globPattern)
107
+
}
108
+
}
109
+
// if no watch block in file
110
+
if !watchSection {
111
+
patterns = append(patterns, glob.MustCompile("*"))
112
+
}
113
+
return patterns
114
+
}
+14
tests/e2e/commit-graph-branch-selector.test.e2e.js
tests/e2e/repo-commitgraph.test.e2e.js
+14
tests/e2e/commit-graph-branch-selector.test.e2e.js
tests/e2e/repo-commitgraph.test.e2e.js
···
1
1
// @ts-check
2
+
3
+
// @watch start
4
+
// templates/repo/graph.tmpl
5
+
// web_src/css/features/gitgraph.css
6
+
// web_src/js/features/repo-graph.js
7
+
// @watch end
8
+
2
9
import {expect} from '@playwright/test';
3
10
import {test, login_user, load_logged_in_context} from './utils_e2e.js';
4
11
5
12
test.beforeAll(async ({browser}, workerInfo) => {
6
13
await login_user(browser, workerInfo, 'user2');
14
+
});
15
+
16
+
test('Commit graph overflow', async ({page}) => {
17
+
await page.goto('/user2/diff-test/graph');
18
+
await expect(page.getByRole('button', {name: 'Mono'})).toBeInViewport({ratio: 1});
19
+
await expect(page.getByRole('button', {name: 'Color'})).toBeInViewport({ratio: 1});
20
+
await expect(page.locator('.selection.search.dropdown')).toBeInViewport({ratio: 1});
7
21
});
8
22
9
23
test('Switch branch', async ({browser}, workerInfo) => {
+5
tests/e2e/dashboard-ci-status.test.e2e.js
+5
tests/e2e/dashboard-ci-status.test.e2e.js
+6
tests/e2e/e2e_test.go
+6
tests/e2e/e2e_test.go
···
38
38
defer cancel()
39
39
40
40
tests.InitTest(true)
41
+
initChangedFiles()
41
42
testE2eWebRoutes = routers.NormalRoutes()
42
43
43
44
os.Unsetenv("GIT_AUTHOR_NAME")
···
99
100
for _, path := range paths {
100
101
_, filename := filepath.Split(path)
101
102
testname := filename[:len(filename)-len(filepath.Ext(path))]
103
+
104
+
if canSkipTest(path) {
105
+
fmt.Printf("No related changes for test, skipping: %s\n", filename)
106
+
continue
107
+
}
102
108
103
109
t.Run(testname, func(t *testing.T) {
104
110
// Default 2 minute timeout
+7
tests/e2e/example.test.e2e.js
+7
tests/e2e/example.test.e2e.js
+6
tests/e2e/explore.test.e2e.js
+6
tests/e2e/explore.test.e2e.js
···
1
1
// @ts-check
2
2
// document is a global in evaluate, so it's safe to ignore here
3
3
// eslint playwright/no-conditional-in-test: 0
4
+
5
+
// @watch start
6
+
// templates/explore/**
7
+
// web_src/modules/fomantic/**
8
+
// @watch end
9
+
4
10
import {expect} from '@playwright/test';
5
11
import {test} from './utils_e2e.js';
6
12
+7
tests/e2e/issue-comment.test.e2e.js
+7
tests/e2e/issue-comment.test.e2e.js
+6
tests/e2e/markdown-editor.test.e2e.js
+6
tests/e2e/markdown-editor.test.e2e.js
+5
tests/e2e/markup.test.e2e.js
+5
tests/e2e/markup.test.e2e.js
+7
tests/e2e/org-settings.test.e2e.js
+7
tests/e2e/org-settings.test.e2e.js
···
1
1
// @ts-check
2
+
3
+
// @watch start
4
+
// templates/org/team/new.tmpl
5
+
// web_src/css/form.css
6
+
// web_src/js/features/org-team.js
7
+
// @watch end
8
+
2
9
import {expect} from '@playwright/test';
3
10
import {test, login_user, login} from './utils_e2e.js';
4
11
import {validate_form} from './shared/forms.js';
+7
tests/e2e/profile_actions.test.e2e.js
+7
tests/e2e/profile_actions.test.e2e.js
+6
tests/e2e/reaction-selectors.test.e2e.js
+6
tests/e2e/reaction-selectors.test.e2e.js
+11
tests/e2e/release.test.e2e.js
+11
tests/e2e/release.test.e2e.js
···
1
1
// @ts-check
2
+
3
+
// @watch start
4
+
// models/repo/attachment.go
5
+
// modules/structs/attachment.go
6
+
// routers/web/repo/**
7
+
// services/attachment/**
8
+
// services/release/**
9
+
// templates/repo/release/**
10
+
// web_src/js/features/repo-release.js
11
+
// @watch end
12
+
2
13
import {expect} from '@playwright/test';
3
14
import {test, login_user, save_visual, load_logged_in_context} from './utils_e2e.js';
4
15
import {validate_form} from './shared/forms.js';
+7
-7
tests/e2e/repo-code.test.e2e.js
+7
-7
tests/e2e/repo-code.test.e2e.js
···
1
1
// @ts-check
2
+
3
+
// @watch start
4
+
// web_src/js/features/repo-code.js
5
+
// web_src/css/repo.css
6
+
// services/gitdiff/**
7
+
// @watch end
8
+
2
9
import {expect} from '@playwright/test';
3
10
import {test, login_user, load_logged_in_context} from './utils_e2e.js';
4
11
···
77
84
}
78
85
}
79
86
});
80
-
81
-
test('Commit graph overflow', async ({page}) => {
82
-
await page.goto('/user2/diff-test/graph');
83
-
await expect(page.getByRole('button', {name: 'Mono'})).toBeInViewport({ratio: 1});
84
-
await expect(page.getByRole('button', {name: 'Color'})).toBeInViewport({ratio: 1});
85
-
await expect(page.locator('.selection.search.dropdown')).toBeInViewport({ratio: 1});
86
-
});
+5
tests/e2e/repo-migrate.test.e2e.js
+5
tests/e2e/repo-migrate.test.e2e.js
+9
tests/e2e/repo-settings.test.e2e.js
+9
tests/e2e/repo-settings.test.e2e.js
···
1
1
// @ts-check
2
+
3
+
// @watch start
4
+
// templates/webhook/shared-settings.tmpl
5
+
// templates/repo/settings/**
6
+
// web_src/css/{form,repo}.css
7
+
// web_src/css/modules/grid.css
8
+
// web_src/js/features/comp/WebHookEditor.js
9
+
// @watch end
10
+
2
11
import {expect} from '@playwright/test';
3
12
import {test, login_user, login} from './utils_e2e.js';
4
13
import {validate_form} from './shared/forms.js';
+6
tests/e2e/repo-wiki.test.e2e.js
+6
tests/e2e/repo-wiki.test.e2e.js
+6
tests/e2e/webauthn.test.e2e.js
+6
tests/e2e/webauthn.test.e2e.js
···
2
2
// SPDX-License-Identifier: MIT
3
3
// @ts-check
4
4
5
+
// @watch start
6
+
// templates/user/auth/**
7
+
// templates/user/settings/**
8
+
// web_src/js/features/user-**
9
+
// @watch end
10
+
5
11
import {expect} from '@playwright/test';
6
12
import {test, login_user, load_logged_in_context} from './utils_e2e.js';
7
13