loading up the forgejo repo on tangled to test page performance
Merge pull request 'tests(e2e): Refactor various tests' (#5929) from fnetx/e2e-flakiness into forgejo
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/5929
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
+229
-116
-1
Makefile
-1
Makefile
···
716
716
717
717
.PHONY: test-e2e-debugserver
718
718
test-e2e-debugserver: e2e.sqlite.test generate-ini-sqlite
719
-
sed -i s/3003/3000/g tests/sqlite.ini
720
719
GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/sqlite.ini ./e2e.sqlite.test -test.run TestDebugserver -test.timeout 24h
721
720
722
721
.PHONY: bench-sqlite
+2
-1
eslint.config.mjs
+2
-1
eslint.config.mjs
···
1125
1125
...playwright.configs['flat/recommended'].rules,
1126
1126
'playwright/no-conditional-in-test': [0],
1127
1127
'playwright/no-conditional-expect': [0],
1128
-
'playwright/no-networkidle': [0],
1128
+
// allow grouping helper functions with tests
1129
+
'unicorn/consistent-function-scoping': [0],
1129
1130
1130
1131
'playwright/no-skipped-test': [
1131
1132
2,
+8
-6
playwright.config.ts
+8
-6
playwright.config.ts
···
1
1
import {devices, type PlaywrightTestConfig} from '@playwright/test';
2
2
3
-
const BASE_URL = process.env.GITEA_URL?.replace?.(/\/$/g, '') || 'http://localhost:3000';
3
+
const BASE_URL = process.env.GITEA_URL?.replace?.(/\/$/g, '') || 'http://localhost:3003';
4
4
5
5
/**
6
6
* @see https://playwright.dev/docs/test-configuration
···
12
12
13
13
// you can adjust this value locally to match your machine's power,
14
14
// or pass `--workers x` to playwright
15
-
workers: process.env.CI ? 1 : 2,
15
+
workers: 1,
16
16
17
17
/* Maximum time one test can run for. */
18
18
timeout: 30 * 1000,
···
22
22
* Maximum time expect() should wait for the condition to be met.
23
23
* For example in `await expect(locator).toHaveText();`
24
24
*/
25
-
timeout: 2000,
25
+
timeout: 3000,
26
26
},
27
27
28
28
/* Fail the build on CI if you accidentally left test.only in the source code. */
···
30
30
31
31
/* Retry on CI only */
32
32
retries: process.env.CI ? 1 : 0,
33
+
/* fail fast */
34
+
maxFailures: process.env.CI ? 1 : 0,
33
35
34
36
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
35
37
reporter: process.env.CI ? 'list' : [['list'], ['html', {outputFolder: 'tests/e2e/reports/', open: 'never'}]],
···
41
43
locale: 'en-US',
42
44
43
45
/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
44
-
actionTimeout: 2000,
46
+
actionTimeout: 3000,
45
47
46
48
/* Maximum time allowed for navigation, such as `page.goto()`. */
47
49
navigationTimeout: 10 * 1000,
···
95
97
},
96
98
],
97
99
98
-
/* Folder for test artifacts such as screenshots, videos, traces, etc. */
100
+
/* Folder for test artifacts created during test execution such as screenshots, traces, etc. */
99
101
outputDir: 'tests/e2e/test-artifacts/',
100
-
/* Folder for test artifacts such as screenshots, videos, traces, etc. */
102
+
/* Folder for explicit snapshots for visual testing */
101
103
snapshotDir: 'tests/e2e/test-snapshots/',
102
104
} satisfies PlaywrightTestConfig;
+88
tests/e2e/README.md
+88
tests/e2e/README.md
···
175
175
If you know noteworthy tests that can act as an inspiration for new tests,
176
176
please add some details here.
177
177
178
+
### Understanding and waiting for page loads
179
+
180
+
[Waiting for a load state](https://playwright.dev/docs/api/class-frame#frame-wait-for-load-state)
181
+
sound like a convenient way to ensure the page was loaded,
182
+
but it only works once and consecutive calls to it
183
+
(e.g. after clicking a button which should reload a page)
184
+
return immediately without waiting for *another* load event.
185
+
186
+
If you match something which is on both the old and the new page,
187
+
you might succeed before the page was reloaded,
188
+
although the code using a `waitForLoadState` might intuitively suggest
189
+
the page was changed before.
190
+
191
+
Interacting with the page before the reload
192
+
(e.g. by opening a dropdown)
193
+
might then race and result in flaky tests,
194
+
depending on the speed of the hardware running the test.
195
+
196
+
A possible way to test that an interaction worked is by checking for a known change first.
197
+
For example:
198
+
199
+
- you submit a form and you want to check that the content persisted
200
+
- checking for the content directly would succeed even without a page reload
201
+
- check for a success message first (will wait until it appears), then verify the content
202
+
203
+
Alternatively, if you know the backend request that will be made before the reload,
204
+
you can explicitly wait for it:
205
+
206
+
~~~js
207
+
const submitted = page.waitForResponse('/my/backend/post/request');
208
+
await page.locator('button').first().click(); // perform your interaction
209
+
await submitted;
210
+
~~~
211
+
212
+
If the page redirects to another URL,
213
+
you can alternatively use:
214
+
215
+
~~~js
216
+
await page.waitForURL('**/target.html');
217
+
~~~
218
+
219
+
### Only sign in if necessary
220
+
221
+
Signing in takes time and is actually executed step-by-step.
222
+
If your test does not rely on a user account, skip this step.
223
+
224
+
~~~js
225
+
test('For anyone', async ({page}) => {
226
+
await page.goto('/somepage');
227
+
~~~
228
+
229
+
If you need a user account, you can use something like:
230
+
231
+
~~~js
232
+
import {test, login_user, login} from './utils_e2e.ts';
233
+
234
+
test.beforeAll(async ({browser}, workerInfo) => {
235
+
await login_user(browser, workerInfo, 'user2'); // or another user
236
+
});
237
+
238
+
test('For signed users only', async ({browser}, workerInfo) => {
239
+
const page = await login({browser}, workerInfo);
240
+
~~~
241
+
178
242
### Run tests very selectively
179
243
180
244
Browser testing can take some time.
···
264
328
265
329
The patterns are evaluated on a "first-match" basis.
266
330
Under the hood, [gobwas/glob](https://github.com/gobwas/glob) is used.
331
+
332
+
## Grouped retry for interactions
333
+
334
+
Sometimes, it can be necessary to retry certain interactions together.
335
+
Consider the following procedure:
336
+
337
+
1. click to open a dropdown
338
+
2. interact with content in the dropdown
339
+
340
+
When for some reason the dropdown does not open,
341
+
for example because of it taking time to initialize after page load,
342
+
the click will succeed,
343
+
but the depending interaction won't,
344
+
although playwright repeatedly tries to find the content.
345
+
346
+
You can [group statements using toPass]()https://playwright.dev/docs/test-assertions#expecttopass).
347
+
This code retries the dropdown click until the second item is found.
348
+
349
+
~~~js
350
+
await expect(async () => {
351
+
await page.locator('.dropdown').click();
352
+
await page.locator('.dropdown .item').first().click();
353
+
}).toPass();
354
+
~~~
-5
tests/e2e/actions.test.e2e.ts
-5
tests/e2e/actions.test.e2e.ts
···
44
44
const page = await context.newPage();
45
45
46
46
await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0');
47
-
await page.waitForLoadState('networkidle');
48
47
49
48
await page.locator('#workflow_dispatch_dropdown>button').click();
50
49
···
55
54
});
56
55
57
56
await page.locator('#workflow-dispatch-submit').click();
58
-
await page.waitForLoadState('networkidle');
59
57
60
58
await expect(page.getByText('Require value for input "String w/o. default".')).toBeVisible();
61
59
});
···
68
66
const page = await context.newPage();
69
67
70
68
await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0');
71
-
await page.waitForLoadState('networkidle');
72
69
73
70
await page.locator('#workflow_dispatch_dropdown>button').click();
74
71
75
72
await page.type('input[name="inputs[string2]"]', 'abc');
76
73
await page.locator('#workflow-dispatch-submit').click();
77
-
await page.waitForLoadState('networkidle');
78
74
79
75
await expect(page.getByText('Workflow run was successfully requested.')).toBeVisible();
80
76
···
83
79
84
80
test('workflow dispatch box not available for unauthenticated users', async ({page}) => {
85
81
await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0');
86
-
await page.waitForLoadState('networkidle');
87
82
88
83
await expect(page.locator('body')).not.toContainText(workflow_trigger_notification_text);
89
84
});
+2
-3
tests/e2e/dashboard-ci-status.test.e2e.ts
+2
-3
tests/e2e/dashboard-ci-status.test.e2e.ts
···
15
15
const response = await page.goto('/?repo-search-query=test_workflows');
16
16
expect(response?.status()).toBe(200);
17
17
18
-
await page.waitForLoadState('networkidle');
19
-
20
18
const repoStatus = page.locator('.dashboard-repos .repo-owner-name-list > li:nth-child(1) > a:nth-child(2)');
21
-
19
+
// wait for network activity to cease (so status was loaded in frontend)
20
+
await page.waitForLoadState('networkidle'); // eslint-disable-line playwright/no-networkidle
22
21
await expect(repoStatus).toHaveAttribute('href', '/user2/test_workflows/actions', {timeout: 10000});
23
22
await expect(repoStatus).toHaveAttribute('data-tooltip-content', 'Failure');
24
23
});
+1
-1
tests/e2e/issue-comment.test.e2e.ts
+1
-1
tests/e2e/issue-comment.test.e2e.ts
···
56
56
await page.locator('#issue-1 .comment-container a[data-tab-for=markdown-previewer]').click();
57
57
await page.click('#issue-1 .comment-container .save');
58
58
59
-
await page.waitForLoadState('networkidle');
59
+
await page.waitForLoadState();
60
60
61
61
// Edit again and assert that edit tab should be active (and not preview tab)
62
62
await page.click('#issue-1 .comment-container .context-menu');
-1
tests/e2e/markup.test.e2e.ts
-1
tests/e2e/markup.test.e2e.ts
···
8
8
test('markup with #xyz-mode-only', async ({page}) => {
9
9
const response = await page.goto('/user2/repo1/issues/1');
10
10
expect(response?.status()).toBe(200);
11
-
await page.waitForLoadState('networkidle');
12
11
13
12
const comment = page.locator('.comment-body>.markup', {hasText: 'test markup light/dark-mode-only'});
14
13
await expect(comment).toBeVisible();
-1
tests/e2e/profile_actions.test.e2e.ts
-1
tests/e2e/profile_actions.test.e2e.ts
+2
-9
tests/e2e/repo-code.test.e2e.ts
+2
-9
tests/e2e/repo-code.test.e2e.ts
···
5
5
// @watch end
6
6
7
7
import {expect} from '@playwright/test';
8
-
import {test, login_user, load_logged_in_context} from './utils_e2e.ts';
9
-
10
-
test.beforeAll(async ({browser}, workerInfo) => {
11
-
await login_user(browser, workerInfo, 'user2');
12
-
});
8
+
import {test} from './utils_e2e.ts';
13
9
14
10
async function assertSelectedLines(page, nums) {
15
11
const pageAssertions = async () => {
···
33
29
return pageAssertions();
34
30
}
35
31
36
-
test('Line Range Selection', async ({browser}, workerInfo) => {
37
-
const context = await load_logged_in_context(browser, workerInfo, 'user2');
38
-
const page = await context.newPage();
39
-
32
+
test('Line Range Selection', async ({page}) => {
40
33
const filePath = '/user2/repo1/src/branch/master/README.md?display=source';
41
34
42
35
const response = await page.goto(filePath);
+3
-9
tests/e2e/repo-commitgraph.test.e2e.ts
+3
-9
tests/e2e/repo-commitgraph.test.e2e.ts
···
5
5
// @watch end
6
6
7
7
import {expect} from '@playwright/test';
8
-
import {test, login_user, load_logged_in_context} from './utils_e2e.ts';
9
-
10
-
test.beforeAll(async ({browser}, workerInfo) => {
11
-
await login_user(browser, workerInfo, 'user2');
12
-
});
8
+
import {test} from './utils_e2e.ts';
13
9
14
10
test('Commit graph overflow', async ({page}) => {
15
11
await page.goto('/user2/diff-test/graph');
···
18
14
await expect(page.locator('.selection.search.dropdown')).toBeInViewport({ratio: 1});
19
15
});
20
16
21
-
test('Switch branch', async ({browser}, workerInfo) => {
22
-
const context = await load_logged_in_context(browser, workerInfo, 'user2');
23
-
const page = await context.newPage();
17
+
test('Switch branch', async ({page}) => {
24
18
const response = await page.goto('/user2/repo1/graph');
25
19
expect(response?.status()).toBe(200);
26
20
···
29
23
await input.pressSequentially('develop', {delay: 50});
30
24
await input.press('Enter');
31
25
32
-
await page.waitForLoadState('networkidle');
26
+
await page.waitForLoadState();
33
27
34
28
await expect(page.locator('#loading-indicator')).toBeHidden();
35
29
await expect(page.locator('#rel-container')).toBeVisible();
-1
tests/e2e/repo-wiki.test.e2e.ts
-1
tests/e2e/repo-wiki.test.e2e.ts
···
9
9
test(`Search for long titles and test for no overflow`, async ({page}, workerInfo) => {
10
10
test.skip(workerInfo.project.name === 'Mobile Safari', 'Fails as always, see https://codeberg.org/forgejo/forgejo/pulls/5326#issuecomment-2313275');
11
11
await page.goto('/user2/repo1/wiki');
12
-
await page.waitForLoadState('networkidle');
13
12
await page.getByPlaceholder('Search wiki').fill('spaces');
14
13
await page.getByPlaceholder('Search wiki').click();
15
14
// workaround: HTMX listens on keyup events, playwright's fill only triggers the input event
+2
-2
tests/e2e/utils_e2e.ts
+2
-2
tests/e2e/utils_e2e.ts
···
37
37
await page.type('input[name=password]', LOGIN_PASSWORD);
38
38
await page.click('form button.ui.primary.button:visible');
39
39
40
-
await page.waitForLoadState('networkidle');
40
+
await page.waitForLoadState();
41
41
42
42
expect(page.url(), {message: `Failed to login user ${user}`}).toBe(`${workerInfo.project.use.baseURL}/`);
43
43
···
67
67
export async function save_visual(page: Page) {
68
68
// Optionally include visual testing
69
69
if (process.env.VISUAL_TEST) {
70
-
await page.waitForLoadState('networkidle');
70
+
await page.waitForLoadState('domcontentloaded');
71
71
// Mock page/version string
72
72
await page.locator('footer div.ui.left').evaluate((node) => node.innerHTML = 'MOCK');
73
73
await expect(page).toHaveScreenshot({
+9
-4
tests/e2e/webauthn.test.e2e.ts
+9
-4
tests/e2e/webauthn.test.e2e.ts
···
8
8
// @watch end
9
9
10
10
import {expect} from '@playwright/test';
11
-
import {test, create_temp_user} from './utils_e2e.ts';
11
+
import {test, create_temp_user, login_user} from './utils_e2e.ts';
12
12
13
13
test('WebAuthn register & login flow', async ({browser, request}, workerInfo) => {
14
14
test.skip(workerInfo.project.name !== 'chromium', 'Uses Chrome protocol');
···
38
38
await page.getByText('Add security key').click();
39
39
40
40
// Logout.
41
-
await page.locator('div[aria-label="Profile and settings…"]').click();
42
-
await page.getByText('Sign Out').click();
41
+
await expect(async () => {
42
+
await page.locator('div[aria-label="Profile and settings…"]').click();
43
+
await page.getByText('Sign Out').click();
44
+
}).toPass();
43
45
await page.waitForURL(`${workerInfo.project.use.baseURL}/`);
44
46
45
47
// Login.
···
57
59
expect(response?.status()).toBe(200);
58
60
await page.getByRole('button', {name: 'Remove'}).click();
59
61
await page.getByRole('button', {name: 'Yes'}).click();
60
-
await page.waitForURL(`${workerInfo.project.use.baseURL}/user/settings/security`);
62
+
await page.waitForLoadState();
63
+
64
+
// verify the user can login without a key
65
+
await login_user(browser, workerInfo, username);
61
66
});