Publish option (#1424)

* Added scenario for --publish option

* Add --publish option

* Make publish scenario pass with cheating

* Add a ReportServer to the World

* Start TDDing HttpStream

* Write HttpStream in a temporary file

* Pipe tempfile to http request

* Wait for request to finish before closing server

* Capture body on server

* Wait for the server to receive all the body

* Add new test for GET/PUT redirect between lambda and S3

* Follow Location after get

* Fix dependency lint

* Cleanup

* Refactor

* Extract HttpStream

* Extract FakeReportServer

* Use FakeReportServer in publish.feature

* Add failing spec for --publish

* Scenario is passing

* Print response body in errors. Send content-length

* Added tests for outputting a banner from the report server content

* http_stream logs response from server to console

* Enable publishing report with ENV var

* Fix lint error

* Start adding support for CUCUMBER_PUBLISH_TOKEN

* Publish with Authorization header when CUCUMBER_PUBLISH_TOKEN is set

* Restore newline

* Add scenarios describing the banner advertising --publish

* Start implementing banner display when --publish is not set

* Add isPublishing attribute to configuration

* Add test checking is suppressPublicationBanner is set + added --publish-quiet option

* Print banners to stderr, fixed import of cucumber

* Adapt banner to cucumber.js

* Add ANSI colours to banner

* Extract banner code to its own file

* more maintainable publish banner

* revert dep lint config, lint fix banner

* Use arrow function

* Use doesHaveValue in conditionals. Update comment about 3xx redirects

* Use valueOrDefault instead of boolean expression

* Update changelog

Co-authored-by: Seb Rose <[email protected]>
Co-authored-by: Christophe Bliard <[email protected]>
Co-authored-by: Vincent Pretre <[email protected]>
Co-authored-by: Charles Rudolph <[email protected]>

CHANGELOG.md

@@ -4,6 +4,7 @@ Please see [CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CO
 
 #### New Features
 
+* Add `--publish` option to publish reports to [reports.cucumber.io](https://reports.cucumber.io) [#1424](https://github.com/cucumber/cucumber-js/pull/1424)
 * Add support for Gherkin's [Rule/Example syntax](https://cucumber.io/docs/gherkin/reference/#rule)
 * Add `transpose` method to [data table interface](docs/support_files/data_table_interface.md)
 * Add `log` function to world, providing a shorthand to log plain text as [attachment(s)](docs/support_files/attachments.md)

cucumber.js

@@ -7,6 +7,7 @@ const feature = [
   '--format rerun:@rerun.txt',
   '--format usage:usage.txt',
   '--format message:messages.ndjson',
+  '--publish-quiet',
 ].join(' ')
 
 const cck = [
@@ -24,6 +25,7 @@ const FORMATTERS_INCLUDE = [
   'parameter-types',
   'rules',
   'stack-traces',
+  '--publish-quiet',
 ]
 
 const formatters = [
@@ -36,6 +38,7 @@ const formatters = [
   `compatibility/features/{${FORMATTERS_INCLUDE.join(',')}}/*.ts`,
   '--format',
   'message',
+  '--publish-quiet',
 ].join(' ')
 
 module.exports = {

features/publish.feature

@@ -0,0 +1,131 @@
+Feature: Publish reports
+
+  Background:
+    Given a file named "features/a.feature" with:
+      """
+      Feature: a feature
+        Scenario: a scenario
+          Given a step
+      """
+    And a file named "features/step_definitions/steps.js" with:
+      """
+      const {Given} = require('@cucumber/cucumber')
+
+      Given(/^a step$/, function() {})
+      """
+
+  @spawn
+  Scenario: Report is published when --publish is specified
+    Given a report server is running on 'http://localhost:9987'
+    When I run cucumber-js with arguments `--publish` and env `CUCUMBER_PUBLISH_URL=http://localhost:9987/api/reports`
+    Then it passes
+    And the server should receive the following message types:
+      | meta             |
+      | source           |
+      | gherkinDocument  |
+      | pickle           |
+      | stepDefinition   |
+      | testRunStarted   |
+      | testCase         |
+      | testCaseStarted  |
+      | testStepStarted  |
+      | testStepFinished |
+      | testCaseFinished |
+      | testRunFinished  |
+
+  @spawn
+  Scenario: Report is published when CUCUMBER_PUBLISH_ENABLED is set
+    Given a report server is running on 'http://localhost:9987'
+    When I run cucumber-js with arguments `` and env `CUCUMBER_PUBLISH_ENABLED=1 CUCUMBER_PUBLISH_URL=http://localhost:9987/api/reports`
+    Then it passes
+    And the server should receive the following message types:
+      | meta             |
+      | source           |
+      | gherkinDocument  |
+      | pickle           |
+      | stepDefinition   |
+      | testRunStarted   |
+      | testCase         |
+      | testCaseStarted  |
+      | testStepStarted  |
+      | testStepFinished |
+      | testCaseFinished |
+      | testRunFinished  |
+
+  @spawn
+  Scenario: Report is published when CUCUMBER_PUBLISH_TOKEN is set
+    Given a report server is running on 'http://localhost:9987'
+    When I run cucumber-js with arguments `` and env `CUCUMBER_PUBLISH_TOKEN=keyboardcat CUCUMBER_PUBLISH_URL=http://localhost:9987/api/reports`
+    Then it passes
+    And the server should receive the following message types:
+      | meta             |
+      | source           |
+      | gherkinDocument  |
+      | pickle           |
+      | stepDefinition   |
+      | testRunStarted   |
+      | testCase         |
+      | testCaseStarted  |
+      | testStepStarted  |
+      | testStepFinished |
+      | testCaseFinished |
+      | testRunFinished  |
+    And the server should receive an "Authorization" header with value "Bearer keyboardcat"
+
+  @spawn
+  Scenario: a banner is displayed after publication
+    Given a report server is running on 'http://localhost:9987'
+    When I run cucumber-js with arguments `--publish` and env `CUCUMBER_PUBLISH_URL=http://localhost:9987/api/reports`
+    Then the error output contains the text:
+      """
+      ┌──────────────────────────────────────────────────────────────────────────┐
+      │ View your Cucumber Report at:                                            │
+      │ https://reports.cucumber.io/reports/f318d9ec-5a3d-4727-adec-bd7b69e2edd3 │
+      │                                                                          │
+      │ This report will self-destruct in 24h unless it is claimed or deleted.   │
+      └──────────────────────────────────────────────────────────────────────────┘
+      """
+
+  @spawn
+  Scenario: when results are not published, a banner explains how to publish
+    When I run cucumber-js
+    Then the error output contains the text:
+      """
+      ┌──────────────────────────────────────────────────────────────────────────┐
+      │ Share your Cucumber Report with your team at https://reports.cucumber.io │
+      │                                                                          │
+      │ Command line option:    --publish                                        │
+      │ Environment variable:   CUCUMBER_PUBLISH_ENABLED=true                    │
+      │                                                                          │
+      │ More information at https://reports.cucumber.io/docs/cucumber-js         │
+      │                                                                          │
+      │ To disable this message, add this to your ./cucumber.js:                 │
+      │ module.exports = { default: '--publish-quiet' }                          │
+      └──────────────────────────────────────────────────────────────────────────┘
+      """
+  @spawn
+  Scenario: the publication banner is not shown when publication is done
+    When I run cucumber-js with arguments `<args>` and env `<env>`
+    Then the error output does not contain the text:
+      """
+      Share your Cucumber Report with your team at https://reports.cucumber.io
+      """
+
+  Examples:
+    | args      | env                           |
+    | --publish |                               |
+    |           | CUCUMBER_PUBLISH_ENABLED=true |
+    |           | CUCUMBER_PUBLISH_TOKEN=123456 |
+
+  @spawn
+  Scenario: the publication banner is not shown when publication is disabled
+    When I run cucumber-js with arguments `<args>` and env `<env>`
+    Then the error output does not contain the text:
+      """
+      Share your Cucumber Report with your team at https://reports.cucumber.io
+      """
+
+  Examples:
+    | args            | env                         |
+    | --publish-quiet |                             |
+    |                 | CUCUMBER_PUBLISH_QUIET=true |

features/step_definitions/cli_steps.ts

@@ -22,6 +22,27 @@ When(
   }
 )
 
+When(
+  /^I run cucumber-js with arguments `(|.+)` and env `(|.+)`$/,
+  { timeout: 10000 },
+  async function (this: World, args: string, envs: string) {
+    const renderedArgs = Mustache.render(valueOrDefault(args, ''), this)
+    const stringArgs = stringArgv(renderedArgs)
+    const initialValue: NodeJS.ProcessEnv = {}
+    const env: NodeJS.ProcessEnv = (envs === null ? '' : envs)
+      .split(/\s+/)
+      .map((keyValue) => keyValue.split('='))
+      .reduce((dict, pair) => {
+        dict[pair[0]] = pair[1]
+        return dict
+      }, initialValue)
+    return await this.run(this.localExecutablePath, stringArgs, {
+      ...process.env,
+      ...env,
+    })
+  }
+)
+
 When(
   /^I run cucumber-js with all formatters(?: and `(|.+)`)?$/,
   { timeout: 10000 },
@@ -100,6 +121,15 @@ Then(/^the error output contains the text:$/, function (
   expect(actualOutput).to.include(expectedOutput)
 })
 
+Then('the error output does not contain the text:', function (
+  this: World,
+  text: string
+) {
+  const actualOutput = normalizeText(this.lastRun.errorOutput)
+  const expectedOutput = normalizeText(text)
+  expect(actualOutput).not.to.include(expectedOutput)
+})
+
 Then(/^I see the version of Cucumber$/, function (this: World) {
   const actualOutput = this.lastRun.output
   const expectedOutput = `${version as string}\n`

features/step_definitions/report_server_steps.ts

@@ -0,0 +1,42 @@
+import { Given, Then, DataTable } from '../..'
+import { World } from '../support/world'
+import { expect } from 'chai'
+import { URL } from 'url'
+import FakeReportServer from '../../test/fake_report_server'
+import assert from 'assert'
+
+Given('a report server is running on {string}', async function (
+  this: World,
+  url: string
+) {
+  const port = parseInt(new URL(url).port)
+  this.reportServer = new FakeReportServer(port)
+  await this.reportServer.start()
+})
+
+Then('the server should receive the following message types:', async function (
+  this: World,
+  expectedMessageTypesTable: DataTable
+) {
+  const expectedMessageTypes = expectedMessageTypesTable
+    .raw()
+    .map((row) => row[0])
+
+  const receivedBodies = await this.reportServer.stop()
+  const ndjson = receivedBodies.toString('utf-8').trim()
+  if (ndjson === '') assert.fail('Server received nothing')
+
+  const receivedMessageTypes = ndjson
+    .split(/\n/)
+    .map((line) => JSON.parse(line))
+    .map((envelope) => Object.keys(envelope)[0])
+
+  expect(receivedMessageTypes).to.deep.eq(expectedMessageTypes)
+})
+
+Then(
+  'the server should receive a(n) {string} header with value {string}',
+  function (this: World, name: string, value: string) {
+    expect(this.reportServer.receivedHeaders[name.toLowerCase()]).to.eq(value)
+  }
+)

features/support/hooks.ts

@@ -106,3 +106,9 @@ After(function (this: World) {
     )
   }
 })
+
+After(async function (this: World) {
+  if (this.reportServer?.started) {
+    await this.reportServer.stop()
+  }
+})

features/support/world.ts

@@ -10,6 +10,7 @@ import VError from 'verror'
 import _ from 'lodash'
 import ndjsonParse from 'ndjson-parse'
 import { messages } from '@cucumber/messages'
+import FakeReportServer from '../../test/fake_report_server'
 
 interface ILastRun {
   error: any
@@ -32,8 +33,13 @@ export class World {
   public verifiedLastRunError: boolean
   public localExecutablePath: string
   public globalExecutablePath: string
+  public reportServer: FakeReportServer
 
-  async run(executablePath: string, inputArgs: string[]): Promise<void> {
+  async run(
+    executablePath: string,
+    inputArgs: string[],
+    env: NodeJS.ProcessEnv = process.env
+  ): Promise<void> {
     const messageFilename = 'message.ndjson'
     const args = ['node', executablePath]
       .concat(inputArgs, [
@@ -54,9 +60,14 @@ export class World {
 
     if (this.spawn) {
       result = await new Promise((resolve) => {
-        execFile(args[0], args.slice(1), { cwd }, (error, stdout, stderr) => {
-          resolve({ error, stdout, stderr })
-        })
+        execFile(
+          args[0],
+          args.slice(1),
+          { cwd, env },
+          (error, stdout, stderr) => {
+            resolve({ error, stdout, stderr })
+          }
+        )
       })
     } else {
       const stdout = new PassThrough()

package.json

@@ -184,6 +184,7 @@
     "stack-chain": "^2.0.0",
     "stacktrace-js": "^2.0.2",
     "string-argv": "^0.3.1",
+    "tmp": "^0.2.1",
     "util-arity": "^1.1.0",
     "verror": "^1.10.0"
   },
@@ -193,6 +194,7 @@
     "@types/bluebird": "3.5.32",
     "@types/chai": "4.2.12",
     "@types/dirty-chai": "2.0.2",
+    "@types/express": "^4.17.7",
     "@types/fs-extra": "9.0.1",
     "@types/glob": "7.1.3",
     "@types/lodash": "4.14.161",
@@ -228,6 +230,7 @@
     "eslint-plugin-prettier": "3.1.4",
     "eslint-plugin-promise": "4.2.1",
     "eslint-plugin-standard": "4.0.1",
+    "express": "^4.17.1",
     "fs-extra": "9.0.1",
     "mocha": "8.1.3",
     "mustache": "4.0.1",
@@ -241,7 +244,6 @@
     "sinon-chai": "3.5.0",
     "stream-buffers": "3.0.2",
     "stream-to-string": "1.2.0",
-    "tmp": "0.2.1",
     "ts-node": "9.0.0",
     "tsify": "5.0.2",
     "typescript": "4.0.2"

src/cli/argv_parser.ts

@@ -34,6 +34,8 @@ export interface IParsedArgvOptions {
   parallel: number
   predictableIds: boolean
   profile: string[]
+  publish: boolean
+  publishQuiet: boolean
   require: string[]
   requireModule: string[]
   retry: number
@@ -168,6 +170,16 @@ const ArgvParser = {
         'Use predictable ids in messages (option ignored if using parallel)',
         false
       )
+      .option(
+        '--publish',
+        'Publish a report to https://reports.cucumber.io',
+        false
+      )
+      .option(
+        '--publish-quiet',
+        "Don't print information banner about publishing reports",
+        false
+      )
       .option(
         '-r, --require <GLOB|DIR|FILE>',
         'require files before executing features (repeatable)',