Move Playwright tagging from tag to annotation with buildkite.tag. prefix

README.md

@@ -179,13 +179,19 @@ Official [Buildkite Test Engine](https://buildkite.com/platform/test-engine) col
     ],
     ```
 
-    If you would like to pass execution tags through to Test Engine, then you can use Playwright's tagging syntax as follows:
+    If you would like to pass execution tags through to Test Engine, add an annotation with a `buildkite.tag.` prefix:
 
+    ```js
+    test('my test', {
+      annotation: { type: 'buildkite.tag.team', description: 'my-team' },
+    }, async ({ page }) => {
+      ...
+    })
     ```
-    test('has tags', { tag: ['@type:feature'] }, ...)
-    ```
 
-    This will be threaded through to Test Engine as an execution tag with key set to `type` and value set to `feature`.
+    This will be threaded through to Test Engine as an execution tag with key set to `team` and value set to `my-team`.
+
+    > **Deprecated:** passing tags via Playwright's `tag` option (e.g. `test('has tags', { tag: ['@type:feature'] }, ...)`) will be removed in a future release. Please migrate to annotations as shown above.
 
    ### Cypress
 

e2e/playwright.test.js

@@ -11,8 +11,8 @@ const runPlaywright = (args, env) => {
 
   return new Promise((resolve) => {
     const command = `npm test -- ${args.join(' ')}`
-    exec(command, { cwd, env: { ...env, JEST_WORKER_ID: undefined } }, (error, stdout) => {
-      resolve(stdout)
+    exec(command, { cwd, env: { ...env, JEST_WORKER_ID: undefined } }, (error, stdout, stderr) => {
+      resolve(stdout + stderr)
     })
   })
 }
@@ -48,7 +48,7 @@ describe('examples/playwright', () => {
   });
 
   test('it posts the correct JSON', async () => {
-    const stdout = await runPlaywright([], env)
+    const stdout = await runPlaywright(["tests/example.spec.js"], env)
 
     const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m)
     const data = JSON.parse(jsonMatch[1])["data"]
@@ -80,16 +80,33 @@ describe('examples/playwright', () => {
       })
     ]))
 
-    expect(data).toHaveProperty("data[2].tags", { foo: "bar" });
-    expect(data).toHaveProperty("data[3].tags", { foo: "bar", baz: 'qux' });
-    expect(data).toHaveProperty("data[4].tags", {});
-
     expect(stdout).toMatch(/Test Engine .* response/m)
   }, TIMEOUT);
 
+  describe('tagging via annotations', () => {
+    const runTaggingTest = () => runPlaywright(["tests/tag.spec.js"], env)
+
+    test('it correctly parses tags and annotations', async () => {
+      const stdout = await runTaggingTest()
+
+      const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m)
+      const data = JSON.parse(jsonMatch[1])["data"]["data"]
+
+      expect(data.length).toEqual(1)
+      expect(data[0]).toHaveProperty("tags", { team: "new-team", priority: "high", type: "e2e" });
+    }, TIMEOUT);
+
+    test('it warns once when the deprecated tag syntax is used', async () => {
+      const stdout = await runTaggingTest()
+
+      const warnings = stdout.match(/Playwright `tag`.*is deprecated/g) || []
+      expect(warnings.length).toEqual(1)
+    }, TIMEOUT);
+  });
+
   describe('when --retries option is used', () => {
     test("it posts all retried executions", async () => {
-      const stdout = await runPlaywright(["--retries=1"], env)
+      const stdout = await runPlaywright(["tests/example.spec.js", "--retries=1"], env)
 
       const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m)
       const data = JSON.parse(jsonMatch[1])["data"]["data"];
@@ -103,7 +120,7 @@ describe('examples/playwright', () => {
 
   describe('when --timeout is exceeded', () => {
     test("it posts the failures", async () => {
-      const stdout = await runPlaywright(["--timeout=1"], env)
+      const stdout = await runPlaywright(["tests/example.spec.js","--timeout=1"], env)
 
       const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m)
       const data = JSON.parse(jsonMatch[1])["data"];
@@ -145,7 +162,7 @@ describe('examples/playwright', () => {
   })
 
   test('it supports test location prefixes for monorepos', async () => {
-    const stdout = await runPlaywright([], { ...env, BUILDKITE_ANALYTICS_LOCATION_PREFIX: "some-sub-dir/" })
+    const stdout = await runPlaywright(["tests/example.spec.js"], { ...env, BUILDKITE_ANALYTICS_LOCATION_PREFIX: "some-sub-dir/" })
 
     const jsonMatch = stdout.match(/.*Test Engine Sending: ({.*})/m)
     const data = JSON.parse(jsonMatch[1])["data"]

examples/playwright/tests/tag.spec.js

@@ -0,0 +1,14 @@
+const { test, expect } = require('@playwright/test');
+
+test('with tags and annotations', {
+  tag: ['@team:old-team', '@type:e2e'],
+  annotation: [
+    { type: 'buildkite.tag.team', description: 'new-team' },
+    { type: 'buildkite.tag.priority', description: 'high' },
+    { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/23180' },
+  ],
+}, async ({ page }) => {
+  await page.goto('/');
+
+  await expect(page).toHaveTitle(/Buildkite/);
+});

playwright/reporter.js

@@ -28,11 +28,18 @@ class PlaywrightBuildkiteTestEngineReporter {
     this._options = options;
     this._paths = new Paths({ cwd: process.cwd() }, this._testEnv.location_prefix);
     this._tagPattern = /^@[^:]+:[^:]+$/;
+    this._annotationTagPrefix = 'buildkite.tag.';
+    this._usedDeprecatedTags = false;
   }
 
   onBegin() { }
 
   onEnd() {
+    if (this._usedDeprecatedTags) {
+      console.warn('[buildkite-test-collector] Playwright `tag` (e.g. "@key:value") is deprecated for tagging tests.');
+      console.warn('[buildkite-test-collector] Use an `annotation` with a `buildkite.tag.` prefix instead, e.g. `buildkite.tag.team`.');
+    }
+
     return new Promise(resolve => {
       uploadTestResults(this._testEnv, this._tags, this._testResults, this._options, resolve);
     })
@@ -50,13 +57,10 @@ class PlaywrightBuildkiteTestEngineReporter {
     const fileName = this._paths.prefixTestPath(test.location.file);
     const location = [fileName, test.location.line, test.location.column].join(':');
 
-    const filteredTags = test.tags.filter(tag => this._tagPattern.test(tag));
-    const tags = filteredTags.reduce((acc, test) => {
-      const [key, value] = test.slice(1).split(':');
-      acc[key] = value;
-
-      return acc;
-    }, {});
+    const tags = {
+      ...this.tagsFromTags(test),
+      ...this.tagsFromAnnotations(test)
+    };
 
     this._testResults.push({
       'id': test.id,
@@ -76,6 +80,42 @@ class PlaywrightBuildkiteTestEngineReporter {
     });
   }
 
+  /**
+   * @deprecated Tagging via Playwright's `tag` option (`@key:value`) is deprecated.
+   * Use an `annotation` with a `buildkite.tag.` prefix instead, e.g.
+   * `{ annotation: { type: "buildkite.tag.team", description: "my-team" } }`.
+   *
+   * @param {TestCase} test
+   */
+  tagsFromTags(test) {
+    const filteredTags = (test.tags || []).filter(tag => this._tagPattern.test(tag));
+
+    if (filteredTags.length > 0) {
+      this._usedDeprecatedTags = true;
+    }
+
+    return filteredTags.reduce((acc, tag) => {
+      const [key, value] = tag.slice(1).split(':');
+      acc[key] = value;
+
+      return acc;
+    }, {});
+  }
+
+  /**
+   * @param {TestCase} test
+   */
+  tagsFromAnnotations(test) {
+    return (test.annotations || []).reduce((acc, annotation) => {
+      if (!annotation.type.startsWith(this._annotationTagPrefix)) return acc;
+
+      const key = annotation.type.slice(this._annotationTagPrefix.length);
+      acc[key] = annotation.description;
+
+      return acc;
+    }, {});
+  }
+
   testEngineResult(status) {
     // Playwright test statuses:
     // - failed