loading up the forgejo repo on tangled to test page performance
tests(e2e): Various fixes to visual testing (#6569)
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6569
Reviewed-by: 0ko <0ko@noreply.codeberg.org>
+34
-20
+1
tests/e2e/actions.test.e2e.ts
+1
tests/e2e/actions.test.e2e.ts
+4
-1
tests/e2e/clipboard-copy.test.e2e.ts
+4
-1
tests/e2e/clipboard-copy.test.e2e.ts
···
8
8
// @watch end
9
9
10
10
import {expect} from '@playwright/test';
11
-
import {test} from './utils_e2e.ts';
11
+
import {save_visual, test} from './utils_e2e.ts';
12
12
13
13
test('copy src file path to clipboard', async ({page}, workerInfo) => {
14
14
test.skip(['Mobile Safari', 'webkit'].includes(workerInfo.project.name), 'Apple clipboard API addon - starting at just $499!');
···
19
19
await page.click('[data-clipboard-text]');
20
20
const clipboardText = await page.evaluate(() => navigator.clipboard.readText());
21
21
expect(clipboardText).toContain('README.md');
22
+
await save_visual(page);
22
23
});
23
24
24
25
test('copy diff file path to clipboard', async ({page}, workerInfo) => {
···
30
31
await page.click('[data-clipboard-text]');
31
32
const clipboardText = await page.evaluate(() => navigator.clipboard.readText());
32
33
expect(clipboardText).toContain('README.md');
34
+
await expect(page.getByText('Copied')).toBeVisible();
35
+
await save_visual(page);
33
36
});
+3
-2
tests/e2e/dashboard-ci-status.test.e2e.ts
+3
-2
tests/e2e/dashboard-ci-status.test.e2e.ts
···
3
3
// @watch end
4
4
5
5
import {expect} from '@playwright/test';
6
-
import {save_visual, test} from './utils_e2e.ts';
6
+
import {test} from './utils_e2e.ts';
7
7
8
8
test.use({user: 'user2'});
9
9
···
23
23
const repoStatus = page.locator('.dashboard-repos .repo-owner-name-list > li:nth-child(1) > a:nth-child(2)');
24
24
await expect(repoStatus).toHaveAttribute('href', '/user2/test_workflows/actions', {timeout: 10000});
25
25
await expect(repoStatus).toHaveAttribute('data-tooltip-content', /^(Error|Failure)$/);
26
-
await save_visual(page);
26
+
// ToDo: Ensure stable screenshot of dashboard. Known to be flaky: https://code.forgejo.org/forgejo/visual-browser-testing/commit/206d4cfb7a4af6d8d7043026cdd4d63708798b2a
27
+
// await save_visual(page);
27
28
});
+6
-1
tests/e2e/e2e_test.go
+6
-1
tests/e2e/e2e_test.go
···
82
82
83
83
runArgs := []string{"npx", "playwright", "test"}
84
84
85
+
_, testVisual := os.LookupEnv("VISUAL_TEST")
85
86
// To update snapshot outputs
86
87
if _, set := os.LookupEnv("ACCEPT_VISUAL"); set {
87
88
runArgs = append(runArgs, "--update-snapshots")
···
105
106
onForgejoRun(t, func(*testing.T, *url.URL) {
106
107
defer DeclareGitRepos(t)()
107
108
thisTest := runArgs
109
+
// when all tests are run, use unique artifacts directories per test to preserve artifacts from other tests
110
+
if testVisual {
111
+
thisTest = append(thisTest, "--output=tests/e2e/test-artifacts/"+testname)
112
+
}
108
113
thisTest = append(thisTest, path)
109
114
cmd := exec.Command(runArgs[0], thisTest...)
110
115
cmd.Env = os.Environ()
···
114
119
cmd.Stderr = os.Stderr
115
120
116
121
err := cmd.Run()
117
-
if err != nil {
122
+
if err != nil && !testVisual {
118
123
log.Fatal("Playwright Failed: %s", err)
119
124
}
120
125
})
+2
-1
tests/e2e/example.test.e2e.ts
+2
-1
tests/e2e/example.test.e2e.ts
···
5
5
// @watch end
6
6
7
7
import {expect} from '@playwright/test';
8
-
import {test} from './utils_e2e.ts';
8
+
import {save_visual, test} from './utils_e2e.ts';
9
9
10
10
test('Load Homepage', async ({page}) => {
11
11
const response = await page.goto('/');
···
26
26
expect(page.url()).toBe(`${workerInfo.project.use.baseURL}/`);
27
27
await expect(page.locator('.secondary-nav span>img.ui.avatar')).toBeVisible();
28
28
await expect(page.locator('.ui.positive.message.flash-success')).toHaveText('Account was successfully created. Welcome!');
29
+
await save_visual(page);
29
30
});
30
31
31
32
// eslint-disable-next-line playwright/no-skipped-test
+2
-1
tests/e2e/explore.test.e2e.ts
+2
-1
tests/e2e/explore.test.e2e.ts
···
7
7
// @watch end
8
8
9
9
import {expect} from '@playwright/test';
10
-
import {test} from './utils_e2e.ts';
10
+
import {save_visual, test} from './utils_e2e.ts';
11
11
12
12
test('Explore view taborder', async ({page}) => {
13
13
await page.goto('/explore/repos');
···
42
42
}
43
43
}
44
44
expect(res).toBe(exp);
45
+
await save_visual(page);
45
46
});
+2
-1
tests/e2e/markup.test.e2e.ts
+2
-1
tests/e2e/markup.test.e2e.ts
···
3
3
// @watch end
4
4
5
5
import {expect} from '@playwright/test';
6
-
import {test} from './utils_e2e.ts';
6
+
import {save_visual, test} from './utils_e2e.ts';
7
7
8
8
test('markup with #xyz-mode-only', async ({page}) => {
9
9
const response = await page.goto('/user2/repo1/issues/1');
···
13
13
await expect(comment).toBeVisible();
14
14
await expect(comment.locator('[src$="#gh-light-mode-only"]')).toBeVisible();
15
15
await expect(comment.locator('[src$="#gh-dark-mode-only"]')).toBeHidden();
16
+
await save_visual(page);
16
17
});
+2
tests/e2e/repo-code.test.e2e.ts
+2
tests/e2e/repo-code.test.e2e.ts
···
49
49
// out-of-bounds end line
50
50
await page.goto(`${filePath}#L1-L100`);
51
51
await assertSelectedLines(page, ['1', '2', '3']);
52
+
await save_visual(page);
52
53
});
53
54
54
55
test('Readable diff', async ({page}, workerInfo) => {
···
75
76
await expect(page.getByText(thisDiff.added, {exact: true})).toHaveCSS('background-color', 'rgb(134, 239, 172)');
76
77
}
77
78
}
79
+
await save_visual(page);
78
80
});
79
81
80
82
test.describe('As authenticated user', () => {
+2
-1
tests/e2e/repo-commitgraph.test.e2e.ts
+2
-1
tests/e2e/repo-commitgraph.test.e2e.ts
···
5
5
// @watch end
6
6
7
7
import {expect} from '@playwright/test';
8
-
import {test} from './utils_e2e.ts';
8
+
import {save_visual, test} from './utils_e2e.ts';
9
9
10
10
test('Commit graph overflow', async ({page}) => {
11
11
await page.goto('/user2/diff-test/graph');
···
28
28
await expect(page.locator('#loading-indicator')).toBeHidden();
29
29
await expect(page.locator('#rel-container')).toBeVisible();
30
30
await expect(page.locator('#rev-container')).toBeVisible();
31
+
await save_visual(page);
31
32
});
+2
-1
tests/e2e/repo-migrate.test.e2e.ts
+2
-1
tests/e2e/repo-migrate.test.e2e.ts
···
21
21
await form.locator('button.primary').click({timeout: 5000});
22
22
await expect(page).toHaveURL('user2/invalidrepo');
23
23
await save_visual(page);
24
-
// page screenshot of unauthenticatedPage is checked automatically after the test
25
24
26
25
const ctx = await test_context(browser);
27
26
const unauthenticatedPage = await ctx.newPage();
···
37
36
await save_visual(page);
38
37
await deleteModal.getByRole('button', {name: 'Delete repository'}).click();
39
38
await expect(page).toHaveURL('/');
39
+
// checked last to preserve the order of screenshots from first run
40
+
await save_visual(unauthenticatedPage);
40
41
});
+3
-1
tests/e2e/repo-wiki.test.e2e.ts
+3
-1
tests/e2e/repo-wiki.test.e2e.ts
···
4
4
// @watch end
5
5
6
6
import {expect} from '@playwright/test';
7
-
import {test} from './utils_e2e.ts';
7
+
import {save_visual, test} from './utils_e2e.ts';
8
8
9
9
for (const searchTerm of ['space', 'consectetur']) {
10
10
for (const width of [null, 2560, 4000]) {
···
23
23
await page.getByPlaceholder('Search wiki').dispatchEvent('keyup');
24
24
// timeout is necessary because HTMX search could be slow
25
25
await expect(page.locator('#wiki-search a[href]')).toBeInViewport({ratio: 1});
26
+
await save_visual(page);
26
27
});
27
28
}
28
29
}
···
36
37
// so we manually "type" the last letter
37
38
await page.getByPlaceholder('Search wiki').dispatchEvent('keyup');
38
39
await expect(page.locator('#wiki-search a[href] b')).toHaveText('Page With Spaced Name');
40
+
await save_visual(page);
39
41
});
+1
-9
tests/e2e/utils_e2e.ts
+1
-9
tests/e2e/utils_e2e.ts
···
26
26
},
27
27
user: null,
28
28
authScope: 'shared',
29
-
// see https://playwright.dev/docs/test-fixtures#adding-global-beforeeachaftereach-hooks
30
-
forEachTest: [async ({page}, use) => {
31
-
await use();
32
-
// some tests create a new page which is not yet available here
33
-
// only operate on tests that make the URL available
34
-
if (page.url() !== 'about:blank') {
35
-
await save_visual(page);
36
-
}
37
-
}, {auto: true}],
38
29
});
39
30
40
31
export async function test_context(browser: Browser, options?: BrowserContextOptions) {
···
128
119
// update order of recently created repos is not fully deterministic
129
120
page.locator('.flex-item-main').filter({hasText: 'relative time in repo'}),
130
121
page.locator('#activity-feed'),
122
+
page.locator('#user-heatmap'),
131
123
// dynamic IDs in fixed-size inputs
132
124
page.locator('input[value*="dyn-id-"]'),
133
125
],