feat(junit): Adds JUnit test result renderer

docs/command-line-arguments.md

@@ -47,6 +47,8 @@ yarn loki test -- --port 9009
 | **`--passWithNoStories`**               | If no stories are detected consider this a success.                                                                                             | _False_                                                   |
 | **`--verboseRenderer`**                 | Plain text renderer, useful for CI.                                                                                                             | _False, true for CI_                                      |
 | **`--silent`**                          | Plain text renderer that will only output errors.                                                                                               | `false`                                                   |
+| **`--junitRenderer`**                   | Generates a JUnit XML report of the test results.                                                                                               | `false`                                                   |
+| **`--junitOutputFile`**                 | Path to the JUnit XML report file.                                                                                                              | `loki-junit-report.xml`                                   |
 | **`--dockerWithSudo`**                  | Run docker commands with sudo.                                                                                                                  | `false`                                                   |
 | **`--dockerNet`**                       | Argument to pass to docker --net, e.g. `host` or `bridge`.                                                                                      | _None_                                                    |
 | **`--device`**                          | Device/emulator ID to target for screenshots. Useful when running multiple devices on a single machine.                                         | _None_                                                    |

docs/continuous-integration.md

@@ -12,3 +12,15 @@ build-storybook && loki --requireReference --reactUri file:./storybook-static
 ```
 
 See the [loki react example project](https://github.com/oblador/loki/tree/master/examples/react) for a reference implementation of this approach.
+
+## JUnit Reporting
+
+For better integration with CI/CD systems, Loki supports generating JUnit XML reports of test results. This is particularly useful for visualizing test results in CI systems like Jenkins, CircleCI, or GitHub Actions.
+
+To enable JUnit reporting, add the `--junitRenderer` flag:
+
+```
+build-storybook && loki --requireReference --reactUri file:./storybook-static --junitRenderer --junitOutputFile=./test-results/loki-report.xml
+```
+
+For more details about the JUnit reporter, see the [JUnit Reporter](junit-reporter.md) documentation.

docs/junit-reporter.md

@@ -0,0 +1,173 @@
+---
+id: junit-reporter
+title: JUnit Reporter
+---
+
+# JUnit Reporter
+
+Loki supports generating JUnit XML reports for test results, which can be integrated with CI/CD systems like Jenkins, CircleCI, GitHub Actions, or any system that understands JUnit XML format.
+
+## Usage
+
+To generate a JUnit XML report while running your visual regression tests, use the `--junitRenderer` flag:
+
+```bash
+loki test --junitRenderer
+```
+
+By default, this will create a `loki-junit-report.xml` file in your project root.
+
+## Configuration
+
+### Command Line Options
+
+| Option | Description | Default |
+|--------|-------------|---------|
+| `--junitRenderer` | Enables JUnit XML report generation | `false` |
+| `--junitOutputFile` | Path to the output JUnit XML file | `loki-junit-report.xml` |
+
+Example with custom output file:
+
+```bash
+loki test --junitRenderer --junitOutputFile=./reports/loki-results.xml
+```
+
+### Configuration in Package.json
+
+You can also configure the JUnit reporter in your `package.json` file:
+
+```json
+{
+  "loki": {
+    "configurations": {
+      // your configurations...
+    },
+    "junitRenderer": true,
+    "junitOutputFile": "./test-results/loki-report.xml"
+  }
+}
+```
+
+Or in your `.lokirc`, `.lokirc.json`, or `loki.config.js` file:
+
+```js
+// loki.config.js
+module.exports = {
+  chromeSelector: '.wrapper > *, #root > *, .story-decorator > *',
+  // other configurations...
+  junitRenderer: true,
+  junitOutputFile: "./test-results/loki-report.xml"
+};
+```
+
+## Output Format
+
+The JUnit XML report includes:
+
+- A test suite named "Loki Visual Tests"
+- Individual test cases for each story, grouped by target and configuration
+- Test case names derived from story kind and name
+- Failure messages for any tests that don't match the reference images
+
+Example output:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="Loki Visual Tests" tests="6" failures="1" errors="0" skipped="0">
+    <testcase classname="chrome.docker.chrome.laptop" name="Button - Primary">
+    </testcase>
+    <testcase classname="chrome.docker.chrome.laptop" name="Button - Secondary">
+      <failure message="Visual difference detected">Visual difference detected</failure>
+    </testcase>
+    <!-- Additional test cases... -->
+  </testsuite>
+</testsuites>
+```
+
+## CI Integration
+
+### Jenkins
+
+For Jenkins, you can use the JUnit plugin to display test results:
+
+```groovy
+// Jenkinsfile
+pipeline {
+  // ...
+  stages {
+    stage('Visual Regression Tests') {
+      steps {
+        sh 'npm run loki:test -- --junitRenderer --junitOutputFile=./loki-results.xml'
+      }
+      post {
+        always {
+          junit 'loki-results.xml'
+        }
+      }
+    }
+  }
+}
+```
+
+### GitLab CI
+
+For GitLab CI, you can use the JUnit report artifact feature to display test results:
+
+```yaml
+# .gitlab-ci.yml
+visual-regression-tests:
+  stage: test
+  image: node:16
+  script:
+    - npm ci
+    - npm run build-storybook
+    - npm run loki:test -- --junitRenderer --junitOutputFile=./loki-junit-report.xml
+  artifacts:
+    when: always
+    paths:
+      - .loki/difference/
+    reports:
+      junit: loki-junit-report.xml
+```
+
+This configuration will:
+- Run your visual tests
+- Show test results in the GitLab CI interface under the Tests tab
+- Upload difference images as artifacts when tests fail
+
+### GitHub Actions
+
+For GitHub Actions, you can use the built-in support for JUnit reports:
+
+```yaml
+# .github/workflows/visual-tests.yml
+jobs:
+  visual-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '16'
+      - name: Install dependencies
+        run: npm ci
+      - name: Run visual tests
+        run: npm run loki:test -- --junitRenderer --junitOutputFile=./loki-results.xml
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: 'loki-results.xml'
+```
+
+## Using Multiple Reporters
+
+The JUnit reporter can be used alongside other reporters. For example, you can use both the interactive reporter (default) and the JUnit reporter:
+
+```bash
+loki test --junitRenderer
+```
+
+This will show the interactive UI in the terminal and also generate the JUnit XML report file. 

packages/runner/package.json

@@ -42,6 +42,7 @@
     "fs-extra": "^9.1.0",
     "import-jsx": "^4.0.1",
     "ink": "^3.2.0",
+    "junit-report-builder": "^3.0.0",
     "minimist": "^1.2.0",
     "ramda": "^0.27.1",
     "react": "^17.0.2",

packages/runner/src/commands/test/default-options.json

@@ -19,5 +19,7 @@
   "reactNativePort": "7007",
   "pixelmatch": {},
   "looksSame": {},
-  "gm": {}
+  "gm": {},
+  "junitRenderer": false,
+  "junitOutputFile": "./.loki/junit-report.xml"
 }

packages/runner/src/commands/test/parse-options.js

@@ -12,6 +12,10 @@ function parseOptions(args, config) {
       'dockerWithSudo',
       'chromeDockerWithoutSeccomp',
       'passWithNoStories',
+      'junitRenderer',
+    ],
+    string: [
+      'junitOutputFile',
     ],
   });
 
@@ -50,13 +54,16 @@ function parseOptions(args, config) {
     gm: $('gm'),
     pixelmatch: $('pixelmatch'),
     verboseRenderer: $('verboseRenderer'),
-    silent: $('silent'),
+    junitRenderer: $('junitRenderer'),
+    junitOutputFile: $('junitOutputFile'),
     requireReference: $('requireReference') || ciInfo.isCI,
     updateReference: argv._[0] === 'update',
-    targetFilter: argv.targetFilter,
-    configurationFilter: argv.configurationFilter || argv._[1],
+    targetFilter: $('targetFilter'),
+    configurationFilter: $('configurationFilter'),
     dockerWithSudo: $('dockerWithSudo'),
     chromeDockerWithoutSeccomp: $('chromeDockerWithoutSeccomp'),
+    chromeDockerUseCopy: $('chromeDockerUseCopy'),
+    silent: $('silent'),
     passWithNoStories: $('passWithNoStories'),
     device: $('device'),
   };

packages/runner/src/commands/test/renderers/index.js

@@ -2,6 +2,7 @@ const importJsx = require('import-jsx');
 const { renderVerbose } = require('./verbose');
 const { renderNonInteractive } = require('./non-interactive');
 const { renderSilent } = require('./silent');
+const { renderJUnit } = require('./junit');
 
 const { renderInteractive } = importJsx('./interactive');
 
@@ -10,4 +11,5 @@ module.exports = {
   renderVerbose,
   renderSilent,
   renderNonInteractive,
+  renderJUnit,
 };

packages/runner/src/commands/test/renderers/junit.js

@@ -0,0 +1,66 @@
+/* eslint-disable no-console */
+const fs = require('fs-extra');
+const path = require('path');
+const builder = require('junit-report-builder');
+
+const {
+  EVENT_CHANGE,
+  EVENT_END,
+  STATUS_FAILED,
+  STATUS_SUCCEEDED,
+} = require('../task-runner');
+const { TASK_TYPE_TEST } = require('../constants');
+
+const renderJUnit = (taskRunner, options = {}) => {
+  const outputFile = options.junitOutputFile || 'loki-junit-report.xml';
+  const suite = builder.testSuite().name('Loki Visual Tests');
+  const testCases = new Map();
+
+  const handleChange = (task) => {
+    if (task.meta.type !== TASK_TYPE_TEST) {
+      return;
+    }
+
+    const testId = `${task.meta.target}/${task.meta.configuration}/${task.meta.kind}/${task.meta.story}`;
+
+    if (task.status === STATUS_FAILED || task.status === STATUS_SUCCEEDED) {
+      let testCase;
+
+      if (!testCases.has(testId)) {
+        testCase = suite.testCase()
+          .className(`${task.meta.target}.${task.meta.configuration}`)
+          .name(`${task.meta.kind} - ${task.meta.story}`);
+        testCases.set(testId, testCase);
+      } else {
+        testCase = testCases.get(testId);
+      }
+
+      if (task.status === STATUS_FAILED) {
+        const errorMessage = task.error ? task.error.message : 'Visual difference detected';
+        testCase.failure(errorMessage);
+      }
+    }
+  };
+
+  const handleEnd = () => {
+    // Ensure the output directory exists
+    const outputDir = path.dirname(outputFile);
+    fs.ensureDirSync(outputDir);
+
+    // Write the JUnit XML report
+    builder.writeTo(outputFile);
+    console.log(`JUnit XML report written to ${outputFile}`);
+  };
+
+  taskRunner.on(EVENT_CHANGE, handleChange);
+  taskRunner.on(EVENT_END, handleEnd);
+
+  const stopRendering = () => {
+    taskRunner.removeListener(EVENT_CHANGE, handleChange);
+    taskRunner.removeListener(EVENT_END, handleEnd);
+  };
+
+  return stopRendering;
+};
+
+module.exports = { renderJUnit }; 

packages/runner/src/commands/test/run-tests.js

@@ -26,6 +26,7 @@ const {
   renderVerbose,
   renderNonInteractive,
   renderSilent,
+  renderJUnit,
 } = require('./renderers');
 const {
   TASK_TYPE_TARGET,
@@ -297,11 +298,24 @@ async function runTests(flatConfigurations, options) {
   const render = getRendererForOptions(options);
   const stopRendering = render(tasks);
 
+  let stopJUnitRendering = null;
+  if (options.junitRenderer) {
+    stopJUnitRendering = renderJUnit(tasks, {
+      junitOutputFile: options.junitOutputFile,
+    });
+  }
+
   try {
     await tasks.run(context);
     stopRendering();
+    if (stopJUnitRendering) {
+      stopJUnitRendering();
+    }
   } catch (err) {
     stopRendering();
+    if (stopJUnitRendering) {
+      stopJUnitRendering();
+    }
     await Promise.all(context.activeTargets.map((target) => target.stop()));
     throw err;
   }

website/sidebars.json

@@ -1,7 +1,7 @@
 {
   "docs": {
     "Introduction": ["getting-started"],
-    "API Reference": ["configuration", "command-line-arguments"],
+    "API Reference": ["configuration", "command-line-arguments", "junit-reporter"],
     "Guides": ["continuous-integration", "serverless", "flaky-tests"]
   }
 }