feat(experiments): add draft launchdarkly-experiment-hypothesis-builder skill

README.md

@@ -39,6 +39,7 @@ Agent Skills are modular, text-based playbooks that teach an agent how to perfor
 | Skill | Description |
 |-------|-------------|
 | `experiments/launchdarkly-experiment-setup` | Set up experiments with metrics, treatments, and data collection |
+| `experiments/launchdarkly-experiment-hypothesis-builder` | Coach a strong, testable hypothesis and hand off a pre-resolved config to experiment setup (draft) |
 
 ### Metrics
 

evals/launchdarkly-experiment-hypothesis-builder/prompt.js

@@ -0,0 +1,30 @@
+/**
+ * Prompt function for this suite: returns SKILL.md wrapped in nunjucks `{% raw %}`.
+ *
+ * promptfoo renders every prompt through nunjucks, and this SKILL.md 
+ * uses its own placeholder-hole syntax — `{{measurement:...}}`,
+ * `{{component:hint}}` in the output-contract examples. Nunjucks parses `measurement`
+ * as a variable, hits the `:`, and throws "expected variable end" on all the tests.
+ *
+ * A function prompt does NOT skip the nunjucks pass: promptfoo's renderPrompt 
+ * assigns the function's return value to basePrompt and still calls nunjucks.renderString on it. 
+ * Its own escape hatch,
+ * autoWrapRawIfPartialNunjucks, only fires on *unclosed* tags (`{{` with no `}}`), so
+ * closed-but-invalid expressions like `{{measurement:...}}` go through unprotected.
+ * Wrapping here supplies the `{% raw %}` that helper would have added. renderString then
+ * returns the file byte-for-byte, so results.json still shows the exact skill text.
+ * The tags live only in this in-memory prompt — SKILL.md is not changed.
+ *
+ * The prompt is not what the agent under test sees. The skill is loaded into
+ * `.claude/skills/<slug>/`, and builds the user turn from vars.user_request. This
+ * exists to satisfy promptfoo's requirement that a prompt be defined.
+ */
+const fs = require("node:fs");
+const path = require("node:path");
+
+const SKILL_MD = path.resolve(
+  __dirname,
+  "../../skills/experiments/launchdarkly-experiment-hypothesis-builder/SKILL.md",
+);
+
+module.exports = () => `{% raw %}${fs.readFileSync(SKILL_MD, "utf-8")}{% endraw %}`;

evals/launchdarkly-experiment-hypothesis-builder/promptfooconfig.yaml

@@ -0,0 +1,261 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+#
+# Evaluates launchdarkly-experiment-hypothesis-builder — an advisory skill that
+# coaches a hypothesis and hands off, never writing. The skill
+# returns one JSON object (SKILL.md "Structured output mode (headless)").
+#
+# Run: promptfoo eval -c shared/defaults.yaml -c launchdarkly-experiment-hypothesis-builder/promptfooconfig.yaml
+description: "Headless contract evaluation of the launchdarkly-experiment-hypothesis-builder skill"
+
+prompts:
+  # A function prompt, not the raw SKILL.md — see prompt.js for why.
+  - id: file://./prompt.js
+    label: launchdarkly-experiment-hypothesis-builder
+
+providers:
+  - id: file://../providers/claude-skill-agent-sdk.js
+    label: claude-skill-agent-sdk
+    config:
+      skill_slug: launchdarkly-experiment-hypothesis-builder
+      expose_mcp_tools: false
+      expose_ask_question: false
+      force_skill_invocation: true    # force-load the skill so the eval tests it, not base Claude
+
+defaultTest:
+  assert:
+    # One contract check for every case, driven by that case's `expect` var.
+    # Concatenated with shared/defaults.yaml's output_valid + latency asserts.
+    #
+    # Scoring is binary on purpose: shared/defaults.yaml applies a 0.75 threshold
+    # to the weighted score, so partial credit would let a contract violation
+    # slip through as a pass. Per-check detail goes in `reason` instead.
+    - type: javascript
+      value: |
+        const ROUTES = ['scaffold','rewrite','junk','aa'];
+        const want = (context.vars && context.vars.expect) || {};
+        const text = String((output && (output.response || output.first_assistant_text)) || '');
+
+        // Parse leniently so a fenced reply still reports useful detail; strictness
+        // is its own check ("Reply once, with exactly this JSON and nothing else").
+        const t = text.trim();
+        let payload = null, strict = false;
+        try { payload = JSON.parse(t); strict = true; } catch (e) {
+          const a = t.indexOf('{'), b = t.lastIndexOf('}');
+          if (a !== -1 && b > a) { try { payload = JSON.parse(t.slice(a, b + 1)); } catch (e2) {} }
+        }
+        if (!payload) return { pass: false, score: 0, reason: `no JSON object in reply: ${text.slice(0,160)}` };
+
+        const checks = [];
+        const add = (name, ok, detail) => checks.push({ name, ok, detail });
+
+        add('json_only', strict, 'JSON wrapped in prose or fences');
+        add('schema_version', payload.schema_version === 1, `got ${payload.schema_version}`);
+        add('route_valid', ROUTES.includes(payload.route), `got ${JSON.stringify(payload.route)}`);
+        if (want.route !== undefined) add('route', payload.route === want.route, `expected ${want.route}, got ${JSON.stringify(payload.route)}`);
+
+        if (want.components) {
+          const got = payload.components || {};
+          for (const k of ['change','measurement','rationale']) {
+            if (want.components[k] === undefined) continue;
+            add(`component.${k}`, Boolean(got[k]) === Boolean(want.components[k]), `expected ${want.components[k]}, got ${JSON.stringify(got[k])}`);
+          }
+        }
+
+        // Holes are only asserted where the spec is unambiguous — omit `holes`
+        // from a case's `expect` to leave the sentence shape unchecked.
+        if (want.holes !== undefined) {
+          const found = [];
+          const re = /\{\{\s*(\w+)\s*:[^}]*\}\}/g;
+          let m;
+          while ((m = re.exec(String(payload.hypothesis || ''))) !== null) found.push(m[1]);
+          const got = found.sort().join(','), exp = [...want.holes].sort().join(',');
+          add('holes', got === exp, `expected [${exp}], got [${got}]`);
+        }
+
+        if (want.measurements) {
+          const list = Array.isArray(payload.measurements) ? payload.measurements : null;
+          add('measurements_array', list !== null, `got ${JSON.stringify(payload.measurements)}`);
+          if (list) {
+            if (want.measurements.count !== undefined) add('measurements.count', list.length === want.measurements.count, `expected ${want.measurements.count}, got ${list.length}`);
+            if (want.measurements.min !== undefined) add('measurements.min', list.length >= want.measurements.min, `expected >=${want.measurements.min}, got ${list.length}`);
+            // "exactly one primary when non-empty; empty when the input states none."
+            if (list.length > 0) {
+              const primaries = list.filter(x => x && x.primary).length;
+              add('exactly_one_primary', primaries === 1, `got ${primaries}`);
+            }
+          }
+        }
+
+        if (want.hypothesisEquals !== undefined) {
+          add('hypothesis_exact', String(payload.hypothesis || '').trim() === want.hypothesisEquals.trim(), `got ${JSON.stringify(String(payload.hypothesis || '').slice(0,120))}`);
+        }
+        for (const needle of (want.mustNotContain || [])) {
+          add(`no_echo(${needle})`, !text.includes(needle), 'raw input echoed back');
+        }
+
+        const failed = checks.filter(c => !c.ok);
+        return {
+          pass: failed.length === 0,
+          score: failed.length === 0 ? 1 : 0,
+          reason: failed.length === 0
+            ? `contract ok (${checks.length} checks): route=${payload.route}`
+            : failed.map(c => `${c.name}: ${c.detail}`).join(' | '),
+        };
+      metric: contract
+      weight: 3
+
+tests:
+  # ---- Gate 1: A/A / platform self-test (checked before junk) ----------------
+  # The one case with a fixed output string (SKILL.md "A/A tests").
+  - description: "A/A: routes to the A/A path with the fixed hypothesis, not a built experiment"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        A/A test to validate bucketing.
+      expect:
+        route: aa
+        components: { change: true, measurement: true, rationale: true }
+        holes: []
+        hypothesisEquals: "If we split traffic evenly between two identical variants, then key metrics show no meaningful difference, because the only thing that differs is random assignment."
+
+  # ---- Gate 2: junk — reserved for non-attempts ------------------------------
+  - description: "Junk: gibberish routes to junk with all three components absent"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        asdfgh
+      expect:
+        route: junk
+        components: { change: false, measurement: false, rationale: false }
+        holes: [change, measurement, rationale]
+        measurements: { count: 0 }
+
+  - description: "Junk: a bare link routes to junk"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        https://example.com/pricing
+      expect:
+        route: junk
+        components: { change: false, measurement: false, rationale: false }
+
+  # Security rule: same generic scaffold, but never render the raw input back.
+  - description: "Security: injection routes to junk and is never echoed back"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        <script>alert('xss')</script>
+      expect:
+        route: junk
+        components: { change: false, measurement: false, rationale: false }
+        mustNotContain: ["<script", "alert("]
+
+  # Coherent intent is never junk 
+  - description: "Bare goal: a specific outcome is a measurement with holes, not junk"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        increase checkout completion
+      expect:
+        route: scaffold
+        components: { change: false, measurement: true, rationale: false }
+        holes: [change, rationale]
+        measurements: { count: 1 }
+
+  - description: "Lone rationale: a standalone mechanism scaffolds, not junk"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        because the current option is buried
+      expect:
+        route: scaffold
+        components: { change: false, measurement: false, rationale: true }
+        holes: [change, measurement]
+        measurements: { count: 0 }
+
+  - description: "Vague direction: not a measurement, but still a scaffold not junk"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        grow the business
+      expect:
+        route: scaffold
+        components: { change: false, measurement: false, rationale: false }
+        holes: [change, measurement, rationale]
+        measurements: { count: 0 }
+
+  # ---- Rule 3: unfalsifiable outcomes do not count as a measurement ----------
+  # The rationale here has to be a genuine mechanism, not a restatement of the
+  # change ("because the layout is cleaner" scores rationale: false per the
+  # "Not a restatement" rule), so the case isolates the unfalsifiable rule.
+  - description: "Unfalsifiable outcome: measurement is treated as missing"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        If we redesign the checkout page, then it will do better or as well, because a simpler layout reduces confusion for first-time buyers.
+      expect:
+        route: scaffold
+        components: { change: true, measurement: false, rationale: true }
+        holes: [measurement]
+        measurements: { count: 0 }
+
+  # ---- Rule 4: exactly one primary measurement ------------------------------
+  # `holes` is deliberately NOT asserted here — SKILL.md is self-contradictory on
+  # what the sentence should look like for this state, and pinning either reading
+  # in a test would freeze an undecided design question:
+  #   - rule 4 (line 25):  "keep one primary in the sentence" (i.e. slot filled)
+  #   - line 233:          "surface them in the hole with 'or'" (i.e. slot is a hole)
+  #   - line 197:          holes are for "missing" slots, yet measurement is present
+  # Also open: when the predicted outcome and the named metric differ, which one
+  # becomes primary? Resolve both in SKILL.md, then assert.
+  - description: "Multiple measurements: two distinct outcomes, exactly one primary"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        I want to test a new onboarding checklist. I think it will boost engagement — let's measure it by revenue.
+      expect:
+        route: scaffold
+        components: { change: true, measurement: true, rationale: false }
+        measurements: { min: 2 }
+
+  # ---- The strong state: 3/3 in canonical order, no holes -------------------
+  - description: "Complete hypothesis: all three components, no holes"
+    vars:
+      max_turns: 3
+      user_request: |
+        Reply with one JSON object and nothing else — no prose, no markdown fences.
+
+        Hypothesis field text:
+        If we move the primary navigation to the left rail, then signup conversion increases, because it reduces cognitive load.
+      expect:
+        route: scaffold
+        components: { change: true, measurement: true, rationale: true }
+        holes: []
+        measurements: { count: 1 }

evals/mocks/tool-responses.json

@@ -1,4 +1,18 @@
 {
+  "list-metrics": {
+    "metrics": [
+      { "key": "checkout-conversion", "name": "Checkout conversion", "kind": "custom", "measureType": "occurrence", "successCriteria": "HigherThanBaseline", "tags": ["growth"] },
+      { "key": "signup-completed", "name": "Signup completed", "kind": "custom", "measureType": "occurrence", "successCriteria": "HigherThanBaseline", "tags": ["growth"] },
+      { "key": "page-load-time", "name": "Page load time", "kind": "custom", "measureType": "value", "unit": "ms", "successCriteria": "LowerThanBaseline", "tags": ["performance"] },
+      { "key": "error-rate", "name": "Error rate", "kind": "custom", "measureType": "occurrence", "successCriteria": "LowerThanBaseline", "tags": ["guardrail"] }
+    ],
+    "totalCount": 4,
+    "pageInfo": { "limit": 20, "offset": 0 }
+  },
+  "list-metric-events": {
+    "events": [{ "eventKey": "{{eventKey}}", "count": 1240, "lastSeen": "2026-07-01T00:00:00Z" }],
+    "totalCount": 1
+  },
   "list-flags": {
     "flags": [
       {

evals/package.json

@@ -15,6 +15,8 @@
     "eval:flag-create:single": "promptfoo eval -c shared/defaults.yaml -c launchdarkly-flag-create/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1",
     "eval:flag-command": "promptfoo eval -c shared/defaults.yaml -c launchdarkly-flag-command/promptfooconfig.yaml --env-file .env --no-cache -o launchdarkly-flag-command/results.json",
     "eval:flag-command:single": "promptfoo eval -c shared/defaults.yaml -c launchdarkly-flag-command/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1",
+    "eval:hypothesis-builder": "promptfoo eval -c shared/defaults.yaml -c launchdarkly-experiment-hypothesis-builder/promptfooconfig.yaml --env-file .env --no-cache -o launchdarkly-experiment-hypothesis-builder/results.json",
+    "eval:hypothesis-builder:single": "promptfoo eval -c shared/defaults.yaml -c launchdarkly-experiment-hypothesis-builder/promptfooconfig.yaml --env-file .env --no-cache --filter-first-n 1",
     "eval:all": "node scripts/aggregate.js --run",
     "eval:aggregate": "node scripts/aggregate.js",
     "eval:diff": "node scripts/diff-changed-skills.js",

evals/providers/claude-skill-agent-sdk.js

@@ -23,6 +23,11 @@
  *                            (Read/Grep/Glob/Bash/Edit/Write/...). Default false.
  *   expose_mcp_tools       - Default true. Set false for skills that should never
  *                            call LaunchDarkly MCP tools (routing/advisory skills).
+ *   mcp_tool_allowlist     - Optional array of tool names. When set, expose ONLY
+ *                            these LaunchDarkly tools — e.g. read-only lookups
+ *                            (list and get tools) for an advisory/handoff skill
+ *                            that must never write, so it physically cannot
+ *                            mutate state. Null/unset exposes all tools.
  *   force_skill_invocation - Default false. When true, set initialPrompt to
  *                            `/<skill_slug>` to explicitly invoke the skill via
  *                            slash command. Use for routing/advisory skills whose
@@ -162,6 +167,11 @@ class ClaudeSkillAgentSdk {
     this.exposeMcpTools = config.expose_mcp_tools !== false;
     this.forceSkillInvocation = Boolean(config.force_skill_invocation);
     this.exposeAskQuestion = Boolean(config.expose_ask_question);
+    // When set, expose ONLY these LaunchDarkly tools (by name) — e.g. read-only
+    // tools for an advisory/handoff skill that must never write. Null = expose all.
+    this.mcpToolAllowlist = Array.isArray(config.mcp_tool_allowlist)
+      ? config.mcp_tool_allowlist
+      : null;
 
     const source = resolveSkillSource(this.skillSlug);
     if (!source) {
@@ -211,8 +221,11 @@ class ClaudeSkillAgentSdk {
     let currentTurn = 0;
     const mockState = createMockState();
 
+    const exposedToolDefs = this.mcpToolAllowlist
+      ? toolDefs.filter((def) => this.mcpToolAllowlist.includes(def.name))
+      : toolDefs;
     const mcpTools = this.exposeMcpTools
-      ? toolDefs.map((def) =>
+      ? exposedToolDefs.map((def) =>
           tool(
             def.name,
             def.description,
@@ -298,7 +311,7 @@ class ClaudeSkillAgentSdk {
 
     const allowedMcpToolNames = [];
     if (this.exposeMcpTools) {
-      for (const def of toolDefs) {
+      for (const def of exposedToolDefs) {
         allowedMcpToolNames.push(`mcp__launchdarkly-mocks__${def.name}`);
       }
     }

evals/scripts/_manifest.js

@@ -49,6 +49,12 @@ const SUITES = [
     skillDir: "skills/feature-flags/launchdarkly-flag-command",
     readme: "skills/feature-flags/launchdarkly-flag-command/README.md",
   },
+  {
+    suite: "launchdarkly-experiment-hypothesis-builder",
+    skillKey: "experiments/launchdarkly-experiment-hypothesis-builder",
+    skillDir: "skills/experiments/launchdarkly-experiment-hypothesis-builder",
+    readme: "skills/experiments/launchdarkly-experiment-hypothesis-builder/README.md",
+  },
 ];
 
 /**