diff --git a/README.md b/README.md index 668d092..cf3dab1 100644 --- a/README.md +++ b/README.md @@ -179,13 +179,19 @@ Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine) col ], ``` - If you would like to pass execution tags through to Test Engine, then you can use Playwright's tagging syntax as follows: + If you would like to pass execution tags through to Test Engine, add an annotation with a `buildkite.tag.` prefix: + ```js + test('my test', { + annotation: { type: 'buildkite.tag.team', description: 'my-team' }, + }, async ({ page }) => { + ... + }) ``` - test('has tags', { tag: ['@type:feature'] }, ...) - ``` - This will be threaded through to Test Engine as an execution tag with key set to `type` and value set to `feature`. + This will be threaded through to Test Engine as an execution tag with key set to `team` and value set to `my-team`. + + > **Deprecated:** passing tags via Playwright's `tag` option (e.g. `test('has tags', { tag: ['@type:feature'] }, ...)`) will be removed in a future release. Please migrate to annotations as shown above. ### Cypress diff --git a/e2e/playwright.test.js b/e2e/playwright.test.js index 90aa432..f84d6d0 100644 --- a/e2e/playwright.test.js +++ b/e2e/playwright.test.js @@ -11,8 +11,8 @@ const runPlaywright = (args, env) => { return new Promise((resolve) => { const command = `npm test -- ${args.join(' ')}` - exec(command, { cwd, env: { ...env, JEST_WORKER_ID: undefined } }, (error, stdout) => { - resolve(stdout) + exec(command, { cwd, env: { ...env, JEST_WORKER_ID: undefined } }, (error, stdout, stderr) => { + resolve(stdout + stderr) }) }) } @@ -48,7 +48,7 @@ describe('examples/playwright', () => { }); test('it posts the correct JSON', async () => { - const stdout = await runPlaywright([], env) + const stdout = await runPlaywright(["tests/example.spec.js"], env) const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m) const data = JSON.parse(jsonMatch[1])["data"] @@ -80,16 +80,33 @@ describe('examples/playwright', () => { }) ])) - expect(data).toHaveProperty("data[2].tags", { foo: "bar" }); - expect(data).toHaveProperty("data[3].tags", { foo: "bar", baz: 'qux' }); - expect(data).toHaveProperty("data[4].tags", {}); - expect(stdout).toMatch(/Test Engine .* response/m) }, TIMEOUT); + describe('tagging via annotations', () => { + const runTaggingTest = () => runPlaywright(["tests/tag.spec.js"], env) + + test('it correctly parses tags and annotations', async () => { + const stdout = await runTaggingTest() + + const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m) + const data = JSON.parse(jsonMatch[1])["data"]["data"] + + expect(data.length).toEqual(1) + expect(data[0]).toHaveProperty("tags", { team: "new-team", priority: "high", type: "e2e" }); + }, TIMEOUT); + + test('it warns once when the deprecated tag syntax is used', async () => { + const stdout = await runTaggingTest() + + const warnings = stdout.match(/Playwright `tag`.*is deprecated/g) || [] + expect(warnings.length).toEqual(1) + }, TIMEOUT); + }); + describe('when --retries option is used', () => { test("it posts all retried executions", async () => { - const stdout = await runPlaywright(["--retries=1"], env) + const stdout = await runPlaywright(["tests/example.spec.js", "--retries=1"], env) const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m) const data = JSON.parse(jsonMatch[1])["data"]["data"]; @@ -103,7 +120,7 @@ describe('examples/playwright', () => { describe('when --timeout is exceeded', () => { test("it posts the failures", async () => { - const stdout = await runPlaywright(["--timeout=1"], env) + const stdout = await runPlaywright(["tests/example.spec.js","--timeout=1"], env) const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m) const data = JSON.parse(jsonMatch[1])["data"]; @@ -145,7 +162,7 @@ describe('examples/playwright', () => { }) test('it supports test location prefixes for monorepos', async () => { - const stdout = await runPlaywright([], { ...env, BUILDKITE_ANALYTICS_LOCATION_PREFIX: "some-sub-dir/" }) + const stdout = await runPlaywright(["tests/example.spec.js"], { ...env, BUILDKITE_ANALYTICS_LOCATION_PREFIX: "some-sub-dir/" }) const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m) const data = JSON.parse(jsonMatch[1])["data"] diff --git a/examples/playwright/tests/tag.spec.js b/examples/playwright/tests/tag.spec.js new file mode 100644 index 0000000..ffd72b8 --- /dev/null +++ b/examples/playwright/tests/tag.spec.js @@ -0,0 +1,14 @@ +const { test, expect } = require('@playwright/test'); + +test('with tags and annotations', { + tag: ['@team:old-team', '@type:e2e'], + annotation: [ + { type: 'buildkite.tag.team', description: 'new-team' }, + { type: 'buildkite.tag.priority', description: 'high' }, + { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/23180' }, + ], +}, async ({ page }) => { + await page.goto('/'); + + await expect(page).toHaveTitle(/Buildkite/); +}); diff --git a/playwright/reporter.js b/playwright/reporter.js index 447c134..b426c30 100644 --- a/playwright/reporter.js +++ b/playwright/reporter.js @@ -28,11 +28,18 @@ class PlaywrightBuildkiteTestEngineReporter { this._options = options; this._paths = new Paths({ cwd: process.cwd() }, this._testEnv.location_prefix); this._tagPattern = /^@[^:]+:[^:]+$/; + this._annotationTagPrefix = 'buildkite.tag.'; + this._usedDeprecatedTags = false; } onBegin() { } onEnd() { + if (this._usedDeprecatedTags) { + console.warn('[buildkite-test-collector] Playwright `tag` (e.g. "@key:value") is deprecated for tagging tests.'); + console.warn('[buildkite-test-collector] Use an `annotation` with a `buildkite.tag.` prefix instead, e.g. `buildkite.tag.team`.'); + } + return new Promise(resolve => { uploadTestResults(this._testEnv, this._tags, this._testResults, this._options, resolve); }) @@ -50,13 +57,10 @@ class PlaywrightBuildkiteTestEngineReporter { const fileName = this._paths.prefixTestPath(test.location.file); const location = [fileName, test.location.line, test.location.column].join(':'); - const filteredTags = test.tags.filter(tag => this._tagPattern.test(tag)); - const tags = filteredTags.reduce((acc, test) => { - const [key, value] = test.slice(1).split(':'); - acc[key] = value; - - return acc; - }, {}); + const tags = { + ...this.tagsFromTags(test), + ...this.tagsFromAnnotations(test) + }; this._testResults.push({ 'id': test.id, @@ -76,6 +80,42 @@ class PlaywrightBuildkiteTestEngineReporter { }); } + /** + * @deprecated Tagging via Playwright's `tag` option (`@key:value`) is deprecated. + * Use an `annotation` with a `buildkite.tag.` prefix instead, e.g. + * `{ annotation: { type: "buildkite.tag.team", description: "my-team" } }`. + * + * @param {TestCase} test + */ + tagsFromTags(test) { + const filteredTags = (test.tags || []).filter(tag => this._tagPattern.test(tag)); + + if (filteredTags.length > 0) { + this._usedDeprecatedTags = true; + } + + return filteredTags.reduce((acc, tag) => { + const [key, value] = tag.slice(1).split(':'); + acc[key] = value; + + return acc; + }, {}); + } + + /** + * @param {TestCase} test + */ + tagsFromAnnotations(test) { + return (test.annotations || []).reduce((acc, annotation) => { + if (!annotation.type.startsWith(this._annotationTagPrefix)) return acc; + + const key = annotation.type.slice(this._annotationTagPrefix.length); + acc[key] = annotation.description; + + return acc; + }, {}); + } + testEngineResult(status) { // Playwright test statuses: // - failed