+1

playwright.config.ts

reviewed

··· 103 103 outputDir: 'tests/e2e/test-artifacts/', 104 104 /* Folder for explicit snapshots for visual testing */ 105 105 snapshotDir: 'tests/e2e/test-snapshots/', 106 106 + snapshotPathTemplate: '{snapshotDir}/snapshots/{testFilePath}/{projectName}_{arg}{ext}', 106 107 } satisfies PlaywrightTestConfig;

+35 -14

tests/e2e/README.md

reviewed

··· 155 155 make test-e2e-sqlite#example 156 156 ``` 157 157 158 158 - ### Visual testing 159 159 - 160 160 - > **Warning** 161 161 - > This is not currently used by most Forgejo contributors. 162 162 - > Your help to improve the situation and allow for visual testing is appreciated. 163 163 - 164 164 - Although the main goal of e2e is assertion testing, we have added a framework for visual regression testing. If you are working on front-end features, please use the following: 165 165 - - Check out `main`, `make clean frontend`, and run e2e tests with `VISUAL_TEST=1` to generate outputs. This will initially fail, as no screenshots exist. You can run the e2e tests again to assert that it passes. 166 166 - - Check out your branch, `make clean frontend`, and run e2e tests with `VISUAL_TEST=1`. You should be able to assert that your front-end changes don't break any other tests unintentionally. 167 167 - 168 168 - `VISUAL_TEST=1` will create screenshots in tests/e2e/test-snapshots. The test will fail the first time this is enabled (until we get visual test image persistence figured out), because it will be testing against an empty screenshot folder. 169 169 - 170 170 - `ACCEPT_VISUAL=1` will overwrite the snapshot images with new images. 171 171 - 172 158 173 159 ## Tips and tricks 174 160 ··· 215 201 ~~~js 216 202 await page.waitForURL('**/target.html'); 217 203 ~~~ 204 204 + 205 205 + ### Visual testing 206 206 + 207 207 + Due to size and frequent updates, we do not host screenshots in the Forgejo repository. 208 208 + However, it is good practice to ensure that your test is capable of generating relevant and stable screenshots. 209 209 + Forgejo is regularly tested against visual regressions in a dedicated repository which contains the screenshots: 210 210 + https://code.forgejo.org/forgejo/visual-browser-testing/ 211 211 + 212 212 + For tests that consume only the `page`, 213 213 + screenshots are automatically created at the end of each test. 214 214 + 215 215 + If your test visits different relevant screens or pages during the test, 216 216 + or creates a custom `page` from context 217 217 + (e.g. for tests that require a signed-in user) 218 218 + calling `await save_visual(page);` explicitly in relevant positions is encouraged. 219 219 + 220 220 + Please confirm locally that your screenshots are stable by performing several runs of your test. 221 221 + When screenshots are available and reproducible, 222 222 + check in your test without the screenshots. 223 223 + 224 224 + When your screenshots differ between runs, 225 225 + for example because dynamic elements (e.g. timestamps, commit hashes etc) 226 226 + change between runs, 227 227 + mask these elements in the `save_visual` function in `utils_e2e.ts`. 228 228 + 229 229 + #### Working with screenshots 230 230 + 231 231 + The following environment variables control visual testing: 232 232 + 233 233 + `VISUAL_TEST=1` will create screenshots in tests/e2e/test-snapshots. 234 234 + The test will fail the first time, 235 235 + because the screenshots are not included with Forgejo. 236 236 + Subsequent runs will comopare against your local copy of the screenshots. 237 237 + 238 238 + `ACCEPT_VISUAL=1` will overwrite the snapshot images with new images. 218 239 219 240 ### Only sign in if necessary 220 241

+1 -1

tests/e2e/example.test.e2e.ts

reviewed

··· 31 31 await expect(page.locator('.secondary-nav span>img.ui.avatar')).toBeVisible(); 32 32 await expect(page.locator('.ui.positive.message.flash-success')).toHaveText('Account was successfully created. Welcome!'); 33 33 34 34 - save_visual(page); 34 34 + await save_visual(page); 35 35 }); 36 36 37 37 // eslint-disable-next-line playwright/no-skipped-test

+4 -4

tests/e2e/release.test.e2e.ts

reviewed

··· 41 41 await page.fill('input[name=attachment-new-name-2]', 'Test'); 42 42 await page.fill('input[name=attachment-new-exturl-2]', 'https://forgejo.org/'); 43 43 await page.click('.remove-rel-attach'); 44 44 - save_visual(page); 44 44 + await save_visual(page); 45 45 await page.click('.button.small.primary'); 46 46 47 47 // Validate release page and click edit ··· 53 53 await expect(page.locator('.download[open] li:nth-of-type(2) a')).toHaveAttribute('href', '/user2/repo2/archive/2.0.tar.gz'); 54 54 await expect(page.locator('.download[open] li:nth-of-type(3)')).toContainText('Test'); 55 55 await expect(page.locator('.download[open] li:nth-of-type(3) a')).toHaveAttribute('href', 'https://forgejo.org/'); 56 56 - save_visual(page); 56 56 + await save_visual(page); 57 57 await page.locator('.octicon-pencil').first().click(); 58 58 59 59 // Validate edit page and edit the release ··· 68 68 await expect(page.locator('.attachment_edit:visible')).toHaveCount(4); 69 69 await page.locator('.attachment_edit:visible').nth(2).fill('Test3'); 70 70 await page.locator('.attachment_edit:visible').nth(3).fill('https://gitea.com/'); 71 71 - save_visual(page); 71 71 + await save_visual(page); 72 72 await page.click('.button.small.primary'); 73 73 74 74 // Validate release page and click edit ··· 78 78 await expect(page.locator('.download[open] li:nth-of-type(3) a')).toHaveAttribute('href', 'https://gitea.io/'); 79 79 await expect(page.locator('.download[open] li:nth-of-type(4)')).toContainText('Test3'); 80 80 await expect(page.locator('.download[open] li:nth-of-type(4) a')).toHaveAttribute('href', 'https://gitea.com/'); 81 81 - save_visual(page); 81 81 + await save_visual(page); 82 82 await page.locator('.octicon-pencil').first().click(); 83 83 84 84 // Delete release

+27 -4

tests/e2e/utils_e2e.ts

reviewed

··· 4 4 context: async ({browser}, use) => { 5 5 return use(await test_context(browser)); 6 6 }, 7 7 + // see https://playwright.dev/docs/test-fixtures#adding-global-beforeeachaftereach-hooks 8 8 + forEachTest: [async ({page}, use) => { 9 9 + await use(); 10 10 + // some tests create a new page which is not yet available here 11 11 + // only operate on tests that make the URL available 12 12 + if (page.url() !== 'about:blank') { 13 13 + await save_visual(page); 14 14 + } 15 15 + }, {auto: true}], 7 16 }); 8 17 9 18 async function test_context(browser: Browser, options?: BrowserContextOptions) { ··· 66 75 // Optionally include visual testing 67 76 if (process.env.VISUAL_TEST) { 68 77 await page.waitForLoadState('domcontentloaded'); 69 69 - // Mock page/version string 70 70 - await page.locator('footer div.ui.left').evaluate((node) => node.innerHTML = 'MOCK'); 78 78 + // Mock/replace dynamic content which can have different size (and thus cannot simply be masked below) 79 79 + await page.locator('footer .left-links').evaluate((node) => node.innerHTML = 'MOCK'); 80 80 + // replace timestamps in repos to mask them later down 81 81 + await page.locator('.flex-item-body > relative-time').filter({hasText: /now|minute/}).evaluateAll((nodes) => { 82 82 + for (const node of nodes) node.outerHTML = 'relative time in repo'; 83 83 + }); 84 84 + await page.locator('relative-time').evaluateAll((nodes) => { 85 85 + for (const node of nodes) node.outerHTML = 'time element'; 86 86 + }); 87 87 + // used for instance for security keys 88 88 + await page.locator('absolute-date').evaluateAll((nodes) => { 89 89 + for (const node of nodes) node.outerHTML = 'time element'; 90 90 + }); 71 91 await expect(page).toHaveScreenshot({ 72 92 fullPage: true, 73 93 timeout: 20000, 74 94 mask: [ 75 75 - page.locator('.secondary-nav span>img.ui.avatar'), 76 76 - page.locator('.ui.dropdown.jump.item span>img.ui.avatar'), 95 95 + page.locator('.ui.avatar'), 96 96 + page.locator('.sha'), 97 97 + page.locator('#repo_migrating'), 98 98 + // update order of recently created repos is not fully deterministic 99 99 + page.locator('.flex-item-main').filter({hasText: 'relative time in repo'}), 77 100 ], 78 101 }); 79 102 }