From 37f8412d73e6f2c7481539f66179d163b45933e6 Mon Sep 17 00:00:00 2001 From: Bobby Nichols Date: Tue, 8 Apr 2025 13:13:21 -0700 Subject: [PATCH] feat(junit): Adds JUnit test result renderer --- docs/command-line-arguments.md | 2 + docs/continuous-integration.md | 12 ++ docs/junit-reporter.md | 173 ++++++++++++++++++ packages/runner/package.json | 1 + .../src/commands/test/default-options.json | 4 +- .../runner/src/commands/test/parse-options.js | 13 +- .../src/commands/test/renderers/index.js | 2 + .../src/commands/test/renderers/junit.js | 66 +++++++ .../runner/src/commands/test/run-tests.js | 14 ++ website/sidebars.json | 2 +- yarn.lock | 20 ++ 11 files changed, 304 insertions(+), 5 deletions(-) create mode 100644 docs/junit-reporter.md create mode 100644 packages/runner/src/commands/test/renderers/junit.js diff --git a/docs/command-line-arguments.md b/docs/command-line-arguments.md index 2da9306a..9a187cc4 100644 --- a/docs/command-line-arguments.md +++ b/docs/command-line-arguments.md @@ -47,6 +47,8 @@ yarn loki test -- --port 9009 | **`--passWithNoStories`** | If no stories are detected consider this a success. | _False_ | | **`--verboseRenderer`** | Plain text renderer, useful for CI. | _False, true for CI_ | | **`--silent`** | Plain text renderer that will only output errors. | `false` | +| **`--junitRenderer`** | Generates a JUnit XML report of the test results. | `false` | +| **`--junitOutputFile`** | Path to the JUnit XML report file. | `loki-junit-report.xml` | | **`--dockerWithSudo`** | Run docker commands with sudo. | `false` | | **`--dockerNet`** | Argument to pass to docker --net, e.g. `host` or `bridge`. | _None_ | | **`--device`** | Device/emulator ID to target for screenshots. Useful when running multiple devices on a single machine. | _None_ | diff --git a/docs/continuous-integration.md b/docs/continuous-integration.md index 0dc57f38..ec6a11c8 100644 --- a/docs/continuous-integration.md +++ b/docs/continuous-integration.md @@ -12,3 +12,15 @@ build-storybook && loki --requireReference --reactUri file:./storybook-static ``` See the [loki react example project](https://github.com/oblador/loki/tree/master/examples/react) for a reference implementation of this approach. + +## JUnit Reporting + +For better integration with CI/CD systems, Loki supports generating JUnit XML reports of test results. This is particularly useful for visualizing test results in CI systems like Jenkins, CircleCI, or GitHub Actions. + +To enable JUnit reporting, add the `--junitRenderer` flag: + +``` +build-storybook && loki --requireReference --reactUri file:./storybook-static --junitRenderer --junitOutputFile=./test-results/loki-report.xml +``` + +For more details about the JUnit reporter, see the [JUnit Reporter](junit-reporter.md) documentation. diff --git a/docs/junit-reporter.md b/docs/junit-reporter.md new file mode 100644 index 00000000..a18e8094 --- /dev/null +++ b/docs/junit-reporter.md @@ -0,0 +1,173 @@ +--- +id: junit-reporter +title: JUnit Reporter +--- + +# JUnit Reporter + +Loki supports generating JUnit XML reports for test results, which can be integrated with CI/CD systems like Jenkins, CircleCI, GitHub Actions, or any system that understands JUnit XML format. + +## Usage + +To generate a JUnit XML report while running your visual regression tests, use the `--junitRenderer` flag: + +```bash +loki test --junitRenderer +``` + +By default, this will create a `loki-junit-report.xml` file in your project root. + +## Configuration + +### Command Line Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--junitRenderer` | Enables JUnit XML report generation | `false` | +| `--junitOutputFile` | Path to the output JUnit XML file | `loki-junit-report.xml` | + +Example with custom output file: + +```bash +loki test --junitRenderer --junitOutputFile=./reports/loki-results.xml +``` + +### Configuration in Package.json + +You can also configure the JUnit reporter in your `package.json` file: + +```json +{ + "loki": { + "configurations": { + // your configurations... + }, + "junitRenderer": true, + "junitOutputFile": "./test-results/loki-report.xml" + } +} +``` + +Or in your `.lokirc`, `.lokirc.json`, or `loki.config.js` file: + +```js +// loki.config.js +module.exports = { + chromeSelector: '.wrapper > *, #root > *, .story-decorator > *', + // other configurations... + junitRenderer: true, + junitOutputFile: "./test-results/loki-report.xml" +}; +``` + +## Output Format + +The JUnit XML report includes: + +- A test suite named "Loki Visual Tests" +- Individual test cases for each story, grouped by target and configuration +- Test case names derived from story kind and name +- Failure messages for any tests that don't match the reference images + +Example output: + +```xml + + + + + + + Visual difference detected + + + + +``` + +## CI Integration + +### Jenkins + +For Jenkins, you can use the JUnit plugin to display test results: + +```groovy +// Jenkinsfile +pipeline { + // ... + stages { + stage('Visual Regression Tests') { + steps { + sh 'npm run loki:test -- --junitRenderer --junitOutputFile=./loki-results.xml' + } + post { + always { + junit 'loki-results.xml' + } + } + } + } +} +``` + +### GitLab CI + +For GitLab CI, you can use the JUnit report artifact feature to display test results: + +```yaml +# .gitlab-ci.yml +visual-regression-tests: + stage: test + image: node:16 + script: + - npm ci + - npm run build-storybook + - npm run loki:test -- --junitRenderer --junitOutputFile=./loki-junit-report.xml + artifacts: + when: always + paths: + - .loki/difference/ + reports: + junit: loki-junit-report.xml +``` + +This configuration will: +- Run your visual tests +- Show test results in the GitLab CI interface under the Tests tab +- Upload difference images as artifacts when tests fail + +### GitHub Actions + +For GitHub Actions, you can use the built-in support for JUnit reports: + +```yaml +# .github/workflows/visual-tests.yml +jobs: + visual-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: '16' + - name: Install dependencies + run: npm ci + - name: Run visual tests + run: npm run loki:test -- --junitRenderer --junitOutputFile=./loki-results.xml + - name: Publish Test Report + uses: mikepenz/action-junit-report@v3 + if: always() # always run even if the previous step fails + with: + report_paths: 'loki-results.xml' +``` + +## Using Multiple Reporters + +The JUnit reporter can be used alongside other reporters. For example, you can use both the interactive reporter (default) and the JUnit reporter: + +```bash +loki test --junitRenderer +``` + +This will show the interactive UI in the terminal and also generate the JUnit XML report file. \ No newline at end of file diff --git a/packages/runner/package.json b/packages/runner/package.json index eae647cc..fda33c4d 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -42,6 +42,7 @@ "fs-extra": "^9.1.0", "import-jsx": "^4.0.1", "ink": "^3.2.0", + "junit-report-builder": "^3.0.0", "minimist": "^1.2.0", "ramda": "^0.27.1", "react": "^17.0.2", diff --git a/packages/runner/src/commands/test/default-options.json b/packages/runner/src/commands/test/default-options.json index 92e9f5bd..839515e6 100644 --- a/packages/runner/src/commands/test/default-options.json +++ b/packages/runner/src/commands/test/default-options.json @@ -19,5 +19,7 @@ "reactNativePort": "7007", "pixelmatch": {}, "looksSame": {}, - "gm": {} + "gm": {}, + "junitRenderer": false, + "junitOutputFile": "./.loki/junit-report.xml" } diff --git a/packages/runner/src/commands/test/parse-options.js b/packages/runner/src/commands/test/parse-options.js index beb8389f..f9b875f5 100644 --- a/packages/runner/src/commands/test/parse-options.js +++ b/packages/runner/src/commands/test/parse-options.js @@ -12,6 +12,10 @@ function parseOptions(args, config) { 'dockerWithSudo', 'chromeDockerWithoutSeccomp', 'passWithNoStories', + 'junitRenderer', + ], + string: [ + 'junitOutputFile', ], }); @@ -50,13 +54,16 @@ function parseOptions(args, config) { gm: $('gm'), pixelmatch: $('pixelmatch'), verboseRenderer: $('verboseRenderer'), - silent: $('silent'), + junitRenderer: $('junitRenderer'), + junitOutputFile: $('junitOutputFile'), requireReference: $('requireReference') || ciInfo.isCI, updateReference: argv._[0] === 'update', - targetFilter: argv.targetFilter, - configurationFilter: argv.configurationFilter || argv._[1], + targetFilter: $('targetFilter'), + configurationFilter: $('configurationFilter'), dockerWithSudo: $('dockerWithSudo'), chromeDockerWithoutSeccomp: $('chromeDockerWithoutSeccomp'), + chromeDockerUseCopy: $('chromeDockerUseCopy'), + silent: $('silent'), passWithNoStories: $('passWithNoStories'), device: $('device'), }; diff --git a/packages/runner/src/commands/test/renderers/index.js b/packages/runner/src/commands/test/renderers/index.js index faa82067..1398e5d6 100644 --- a/packages/runner/src/commands/test/renderers/index.js +++ b/packages/runner/src/commands/test/renderers/index.js @@ -2,6 +2,7 @@ const importJsx = require('import-jsx'); const { renderVerbose } = require('./verbose'); const { renderNonInteractive } = require('./non-interactive'); const { renderSilent } = require('./silent'); +const { renderJUnit } = require('./junit'); const { renderInteractive } = importJsx('./interactive'); @@ -10,4 +11,5 @@ module.exports = { renderVerbose, renderSilent, renderNonInteractive, + renderJUnit, }; diff --git a/packages/runner/src/commands/test/renderers/junit.js b/packages/runner/src/commands/test/renderers/junit.js new file mode 100644 index 00000000..cfd0e32b --- /dev/null +++ b/packages/runner/src/commands/test/renderers/junit.js @@ -0,0 +1,66 @@ +/* eslint-disable no-console */ +const fs = require('fs-extra'); +const path = require('path'); +const builder = require('junit-report-builder'); + +const { + EVENT_CHANGE, + EVENT_END, + STATUS_FAILED, + STATUS_SUCCEEDED, +} = require('../task-runner'); +const { TASK_TYPE_TEST } = require('../constants'); + +const renderJUnit = (taskRunner, options = {}) => { + const outputFile = options.junitOutputFile || 'loki-junit-report.xml'; + const suite = builder.testSuite().name('Loki Visual Tests'); + const testCases = new Map(); + + const handleChange = (task) => { + if (task.meta.type !== TASK_TYPE_TEST) { + return; + } + + const testId = `${task.meta.target}/${task.meta.configuration}/${task.meta.kind}/${task.meta.story}`; + + if (task.status === STATUS_FAILED || task.status === STATUS_SUCCEEDED) { + let testCase; + + if (!testCases.has(testId)) { + testCase = suite.testCase() + .className(`${task.meta.target}.${task.meta.configuration}`) + .name(`${task.meta.kind} - ${task.meta.story}`); + testCases.set(testId, testCase); + } else { + testCase = testCases.get(testId); + } + + if (task.status === STATUS_FAILED) { + const errorMessage = task.error ? task.error.message : 'Visual difference detected'; + testCase.failure(errorMessage); + } + } + }; + + const handleEnd = () => { + // Ensure the output directory exists + const outputDir = path.dirname(outputFile); + fs.ensureDirSync(outputDir); + + // Write the JUnit XML report + builder.writeTo(outputFile); + console.log(`JUnit XML report written to ${outputFile}`); + }; + + taskRunner.on(EVENT_CHANGE, handleChange); + taskRunner.on(EVENT_END, handleEnd); + + const stopRendering = () => { + taskRunner.removeListener(EVENT_CHANGE, handleChange); + taskRunner.removeListener(EVENT_END, handleEnd); + }; + + return stopRendering; +}; + +module.exports = { renderJUnit }; \ No newline at end of file diff --git a/packages/runner/src/commands/test/run-tests.js b/packages/runner/src/commands/test/run-tests.js index 954460c8..1759dd37 100644 --- a/packages/runner/src/commands/test/run-tests.js +++ b/packages/runner/src/commands/test/run-tests.js @@ -26,6 +26,7 @@ const { renderVerbose, renderNonInteractive, renderSilent, + renderJUnit, } = require('./renderers'); const { TASK_TYPE_TARGET, @@ -297,11 +298,24 @@ async function runTests(flatConfigurations, options) { const render = getRendererForOptions(options); const stopRendering = render(tasks); + let stopJUnitRendering = null; + if (options.junitRenderer) { + stopJUnitRendering = renderJUnit(tasks, { + junitOutputFile: options.junitOutputFile, + }); + } + try { await tasks.run(context); stopRendering(); + if (stopJUnitRendering) { + stopJUnitRendering(); + } } catch (err) { stopRendering(); + if (stopJUnitRendering) { + stopJUnitRendering(); + } await Promise.all(context.activeTargets.map((target) => target.stop())); throw err; } diff --git a/website/sidebars.json b/website/sidebars.json index 0d8af35a..9a874671 100755 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -1,7 +1,7 @@ { "docs": { "Introduction": ["getting-started"], - "API Reference": ["configuration", "command-line-arguments"], + "API Reference": ["configuration", "command-line-arguments", "junit-reporter"], "Guides": ["continuous-integration", "serverless", "flaky-tests"] } } diff --git a/yarn.lock b/yarn.lock index c96b1b3c..d55aac4e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7803,6 +7803,11 @@ date-fns@^2.29.1: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.29.3.tgz#27402d2fc67eb442b511b70bbdf98e6411cd68a8" integrity sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA== +date-format@4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/date-format/-/date-format-4.0.3.tgz#f63de5dc08dc02efd8ef32bf2a6918e486f35873" + integrity sha512-7P3FyqDcfeznLZp2b+OMitV9Sz2lUnsT87WaTat9nVwqsBkTzPG3lPLNwW3en6F4pHUiWzr6vb8CLhjdK9bcxQ== + dateformat@^3.0.0: version "3.0.3" resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae" @@ -12869,6 +12874,16 @@ jsprim@^1.2.2: array-includes "^3.1.3" object.assign "^4.1.2" +junit-report-builder@^3.0.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/junit-report-builder/-/junit-report-builder-3.2.1.tgz#93067512353c3d47d2dd5913e695b32d3262096c" + integrity sha512-IMCp5XyDQ4YESDE4Za7im3buM0/7cMnRfe17k2X8B05FnUl9vqnaliX6cgOEmPIeWKfJrEe/gANRq/XgqttCqQ== + dependencies: + date-format "4.0.3" + lodash "^4.17.21" + make-dir "^3.1.0" + xmlbuilder "^15.1.1" + junk@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/junk/-/junk-3.1.0.tgz#31499098d902b7e98c5d9b9c80f43457a88abfa1" @@ -20687,6 +20702,11 @@ xmlbuilder@^13.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-13.0.2.tgz#02ae33614b6a047d1c32b5389c1fdacb2bce47a7" integrity sha512-Eux0i2QdDYKbdbA6AM6xE4m6ZTZr4G4xF9kahI2ukSEMCzwce2eX9WlTI5J3s+NU7hpasFsr8hWIONae7LluAQ== +xmlbuilder@^15.1.1: + version "15.1.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5" + integrity sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg== + xmlbuilder@~9.0.1: version "9.0.7" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"