diff --git a/CHANGELOG.md b/CHANGELOG.md index aa449db..24a1f8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added — sense-check time-match + share-basis guardrails (cofounder feedback) + +- **Time-match guardrail.** Players now carry optional `revenue_as_of` / + `share_as_of` dates (`YYYY[-MM[-DD]]`) — set via the `--player` DSL's new + backward-compatible `key=value` suffixes (`rev_date=` / `share_date=`), the + CSV columns, or the MCP `sense_check` row schema. Dates more than ~6 months + apart fire a per-player warning (pairing a current revenue with a stale share + inflates/deflates the implied TAM — the real Cursor Feb-2026-ARR ÷ mid-2025 + share case, ~2x); absent dates add the softer assumptions line "revenue and + share assumed time-matched — un-dated inputs". An `edgar:` revenue threads its + filing's own fiscal-period end into `revenue_as_of` automatically and rejects + a conflicting caller-supplied date. +- **Share-basis guardrail.** Optional `share_basis` per player (`revenue` | + `usage` | unspecified; DSL `share_basis=`, CSV column, MCP enum). `usage` + fires a strong warning — the denominator must be a REVENUE share; a usage/seat + share understates the implied market (the Copilot 42%-seat-share case). + Unspecified adds "shares assumed to be revenue shares" to the assumptions. + Both guardrails are warnings through the existing engine channel — never + silent corrections — so CLI and MCP render them identically; the MODELED (Q20) + label is untouched. Failure modes #11–#12 documented in + `docs/methods/sense-check-defensibility.md`. + ### Changed — the miss→gap flywheel now runs itself (feedback loop fixes) - **A failed market query now automatically becomes ranked demand.** When the diff --git a/backend/src/strata/cli.py b/backend/src/strata/cli.py index 5939da7..cbe036b 100644 --- a/backend/src/strata/cli.py +++ b/backend/src/strata/cli.py @@ -41,7 +41,9 @@ SenseCheckError, SenseCheckResult, normalize_share, + parse_as_of, parse_money, + parse_share_basis, sense_check, share_token_warning, split_player_spec, @@ -106,11 +108,19 @@ [], "--player", "-p", - help="A top player as 'Name|revenue|share' (repeatable). revenue: 31e9 / 3.1b / $300m, " - "or 'edgar:TICKER' to pull a SOURCED revenue from SEC EDGAR; share: 0.15 or 15.", + help="A top player as 'Name|revenue|share[|sourced][|key=value ...]' (repeatable). " + "revenue: 31e9 / 3.1b / $300m, or 'edgar:TICKER' to pull a SOURCED revenue from " + "SEC EDGAR (its fiscal-period end auto-fills rev_date); share: 0.15 or 15. " + "Optional keys: rev_date=YYYY[-MM[-DD]] / share_date=YYYY[-MM[-DD]] (when the " + "revenue / share were measured; >6 months apart warns) and " + "share_basis=revenue|usage (usage/seat shares warn — the method needs a " + "revenue share). Example: 'Cursor|2b|0.18|rev_date=2026-02|share_date=2025-06'.", ) _SC_FILE_OPT = typer.Option( - None, "--players-file", help="CSV with columns: name,revenue,market_share." + None, + "--players-file", + help="CSV with columns: name,revenue,market_share and optional " + "sourced,revenue_as_of,share_as_of,share_basis.", ) _SC_CONCEPT_OPT = typer.Option( None, "--concept", help="Curated concept slug to cross-check the implied TAM against." @@ -446,11 +456,27 @@ def _share_from_token(token: str, ctx: str) -> float: return share -def _edgar_player(name: str, ticker: str, share: float, client: EDGARClient) -> Player: +def _edgar_player( + name: str, + ticker: str, + share: float, + client: EDGARClient, + *, + revenue_as_of: str = "", + share_as_of: str = "", + share_basis: str = "", +) -> Player: """Resolve an ``edgar:TICKER`` revenue token to a SOURCED (but total-company) - revenue via SEC EDGAR (live HTTP).""" + revenue via SEC EDGAR (live HTTP). The filing's own fiscal-period end becomes + ``revenue_as_of`` — a caller-supplied rev_date is rejected, not silently + overridden, so the dates the guardrail compares are never ambiguous.""" if not ticker: raise SenseCheckError(f"empty EDGAR ticker for player {name!r}") + if revenue_as_of: + raise SenseCheckError( + f"{name!r}: rev_date conflicts with an edgar: revenue — the filing's " + f"fiscal-period end fills revenue_as_of automatically" + ) if os.environ.get("STRATA_OFFLINE") == "1": raise SenseCheckError(f"{name!r} needs a live EDGAR lookup but STRATA_OFFLINE=1") try: @@ -461,20 +487,33 @@ def _edgar_player(name: str, ticker: str, share: float, client: EDGARClient) -> f" resolved {name} revenue {usd(rr.revenue)} " f"(EDGAR {rr.concept} FY{rr.fiscal_year}, {rr.entity_name})" ) - return to_player(name, share, rr) + return to_player(name, share, rr, share_as_of=share_as_of, share_basis=share_basis) def _player_from_spec(spec: str, client: EDGARClient) -> Player: """Parse a --player spec. An ``edgar:TICKER`` revenue token resolves to a sourced total-company revenue; any other token is an analyst estimate parsed offline.""" - parts = split_player_spec(spec) - share = _share_from_token(parts[2], spec) - if parts[1].lower().startswith("edgar:"): - return _edgar_player(parts[0], parts[1].split(":", 1)[1].strip(), share, client) - sourced = len(parts) == 4 and parts[3].lower() in SOURCED_TOKENS + ps = split_player_spec(spec) + share = _share_from_token(ps.share_token, spec) + if ps.revenue_token.lower().startswith("edgar:"): + return _edgar_player( + ps.name, + ps.revenue_token.split(":", 1)[1].strip(), + share, + client, + revenue_as_of=ps.revenue_as_of, + share_as_of=ps.share_as_of, + share_basis=ps.share_basis, + ) return Player( - name=parts[0], revenue=parse_money(parts[1]), market_share=share, revenue_sourced=sourced + name=ps.name, + revenue=parse_money(ps.revenue_token), + market_share=share, + revenue_sourced=ps.sourced, + revenue_as_of=ps.revenue_as_of, + share_as_of=ps.share_as_of, + share_basis=ps.share_basis, ) @@ -496,9 +535,22 @@ def _collect_players(specs: list[str], players_file: Path | None) -> list[Player share = _share_from_token( (row.get("market_share") or "").strip(), f"CSV row {name!r}" ) + raw_rev_as_of = (row.get("revenue_as_of") or "").strip() + revenue_as_of = parse_as_of(raw_rev_as_of) if raw_rev_as_of else "" + raw_share_as_of = (row.get("share_as_of") or "").strip() + share_as_of = parse_as_of(raw_share_as_of) if raw_share_as_of else "" + share_basis = parse_share_basis((row.get("share_basis") or "").strip()) if rev.lower().startswith("edgar:"): players.append( - _edgar_player(name, rev.split(":", 1)[1].strip(), share, client) + _edgar_player( + name, + rev.split(":", 1)[1].strip(), + share, + client, + revenue_as_of=revenue_as_of, + share_as_of=share_as_of, + share_basis=share_basis, + ) ) continue sourced = (row.get("sourced") or "").strip().lower() in SOURCED_TOKENS @@ -508,6 +560,9 @@ def _collect_players(specs: list[str], players_file: Path | None) -> list[Player revenue=parse_money(rev), market_share=share, revenue_sourced=sourced, + revenue_as_of=revenue_as_of, + share_as_of=share_as_of, + share_basis=share_basis, ) ) return players diff --git a/backend/src/strata/mcp/schemas.py b/backend/src/strata/mcp/schemas.py index 409b5db..d164fa3 100644 --- a/backend/src/strata/mcp/schemas.py +++ b/backend/src/strata/mcp/schemas.py @@ -577,6 +577,37 @@ "type": "number", "description": "Estimated revenue share: fraction (0.15) or percent (15).", }, + "revenue_as_of": { + "type": "string", + "description": ( + "When the revenue was measured: 'YYYY', 'YYYY-MM', or " + "'YYYY-MM-DD'. With edgar_ticker it is filled automatically " + "from the filing's fiscal-period end (do not pass both). A " + ">~6-month gap vs share_as_of fires a time-match warning — " + "pairing a current revenue with a stale share inflates/" + "deflates the implied TAM." + ), + }, + "share_as_of": { + "type": "string", + "description": ( + "When the market-share estimate was measured: 'YYYY', " + "'YYYY-MM', or 'YYYY-MM-DD'. See revenue_as_of for the " + "time-match guardrail; un-dated inputs get an assumptions " + "note instead." + ), + }, + "share_basis": { + "type": "string", + "enum": ["revenue", "usage"], + "description": ( + "What the share measures. The method's denominator must be a " + "REVENUE share; 'usage' (seats/usage) triggers a strong " + "warning because a usage share understates the implied " + "market. Unspecified is noted as 'shares assumed to be " + "revenue shares'." + ), + }, }, "required": ["name", "market_share"], "additionalProperties": False, diff --git a/backend/src/strata/mcp/sense_check_tool.py b/backend/src/strata/mcp/sense_check_tool.py index ddebaff..f8ed29c 100644 --- a/backend/src/strata/mcp/sense_check_tool.py +++ b/backend/src/strata/mcp/sense_check_tool.py @@ -26,6 +26,8 @@ Player, SenseCheckError, normalize_share, + parse_as_of, + parse_share_basis, ) from ..sizing.sense_check import ( sense_check as run_sense_check, @@ -52,6 +54,17 @@ def _player_from_row(row: dict[str, Any], live_enabled: bool, client: EDGARClien ) share = normalize_share(float(raw_share)) + # The guardrail inputs are validated at the door so a bad token is an honest + # CODE_INVALID_INPUT, not a deep engine error. + raw_rev_as_of = str(row.get("revenue_as_of") or "").strip() + raw_share_as_of = str(row.get("share_as_of") or "").strip() + try: + revenue_as_of = parse_as_of(raw_rev_as_of) if raw_rev_as_of else "" + share_as_of = parse_as_of(raw_share_as_of) if raw_share_as_of else "" + share_basis = parse_share_basis(str(row.get("share_basis") or "")) + except SenseCheckError as exc: + raise ToolError(f"player {name!r}: {exc}", code=CODE_INVALID_INPUT) from exc + ticker = str(row.get("edgar_ticker") or "").strip() revenue = row.get("revenue") if ticker and revenue is not None: @@ -60,6 +73,12 @@ def _player_from_row(row: dict[str, Any], live_enabled: bool, client: EDGARClien code=CODE_INVALID_INPUT, ) if ticker: + if revenue_as_of: + raise ToolError( + f"player {name!r}: 'revenue_as_of' conflicts with 'edgar_ticker' — the " + f"filing's fiscal-period end fills revenue_as_of automatically", + code=CODE_INVALID_INPUT, + ) if not live_enabled: raise ToolError( f"player {name!r}: resolving 'edgar_ticker' is a live SEC EDGAR call, " @@ -72,13 +91,20 @@ def _player_from_row(row: dict[str, Any], live_enabled: bool, client: EDGARClien rr = resolve_revenue(ticker, client=client) except RevenueLookupError as exc: raise ToolError(str(exc), code=CODE_INVALID_INPUT, retriable=True) from exc - return to_player(name, share, rr) + return to_player(name, share, rr, share_as_of=share_as_of, share_basis=share_basis) if not isinstance(revenue, (int, float)) or isinstance(revenue, bool): raise ToolError( f"player {name!r}: provide a numeric 'revenue' or an 'edgar_ticker'", code=CODE_INVALID_INPUT, ) - return Player(name=name, revenue=float(revenue), market_share=share) + return Player( + name=name, + revenue=float(revenue), + market_share=share, + revenue_as_of=revenue_as_of, + share_as_of=share_as_of, + share_basis=share_basis, + ) def sense_check_tool( @@ -136,6 +162,9 @@ def sense_check_tool( "implied_tam": p.implied_tam, "revenue_sourced": p.revenue_sourced, "revenue_note": p.revenue_note or None, + "revenue_as_of": p.revenue_as_of or None, + "share_as_of": p.share_as_of or None, + "share_basis": p.share_basis or None, } for p in r.per_player ], diff --git a/backend/src/strata/sizing/revenue_lookup.py b/backend/src/strata/sizing/revenue_lookup.py index a05d6b5..83ef4ae 100644 --- a/backend/src/strata/sizing/revenue_lookup.py +++ b/backend/src/strata/sizing/revenue_lookup.py @@ -55,18 +55,33 @@ class ResolvedRevenue: fiscal_year: int | None concept: str source_url: str - - -def to_player(name: str, share: float, resolved: ResolvedRevenue) -> Player: + # The fact's own fiscal-period end (ISO date) — the filing already knows + # WHEN its revenue is measured, so the time-match guardrail gets it for free. + period_end: str = "" + + +def to_player( + name: str, + share: float, + resolved: ResolvedRevenue, + *, + share_as_of: str = "", + share_basis: str = "", +) -> Player: """The ONE way an EDGAR figure becomes a sense-check player: always sourced, always carrying the total-company marker so the engine emits the caveat — - a sourced-but-total figure is never an unmarked sourced line.""" + a sourced-but-total figure is never an unmarked sourced line. The filing's + fiscal-period end auto-threads into ``revenue_as_of`` (time-match guardrail); + the share's date/basis stay caller-owned (the share is an analyst estimate).""" return Player( name=name, revenue=resolved.revenue, market_share=share, revenue_sourced=True, revenue_note=REVENUE_NOTE_FILING_TOTAL_CO, + revenue_as_of=resolved.period_end, + share_as_of=share_as_of, + share_basis=share_basis, ) @@ -122,6 +137,7 @@ def resolve_revenue(ticker: str, *, client: EDGARClient | None = None) -> Resolv fiscal_year=fiscal_year_from_end(fact), concept=concept_tag, source_url=f"https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}", + period_end=fact.end, ) finally: if own: diff --git a/backend/src/strata/sizing/sense_check.py b/backend/src/strata/sizing/sense_check.py index b417e40..9fb25b8 100644 --- a/backend/src/strata/sizing/sense_check.py +++ b/backend/src/strata/sizing/sense_check.py @@ -32,6 +32,7 @@ from __future__ import annotations +import re import statistics from dataclasses import dataclass, field @@ -48,6 +49,22 @@ # a filing-backed revenue — one list shared by every parsing surface. SOURCED_TOKENS = ("sourced", "filing", "true", "1", "yes") +# Time-match guardrail: a revenue and a share dated more than this many months +# apart get a per-player warning — pairing a current revenue with a stale share +# inflates/deflates the implied TAM (validated in practice: Cursor's Feb-2026 +# ARR over its mid-2025 share inflated the implied market ~2x). +TIME_MATCH_WARN_MONTHS = 6 +# The only declarable share bases. 'usage' triggers a strong warning: dividing +# revenue by a seat/usage share understates the implied market (a high-priced +# player's revenue share exceeds its seat share, and vice versa). +SHARE_BASIS_VALUES = ("revenue", "usage") +# Conditional assumptions-block lines — softer than warnings, still explicit. +NOTE_UNDATED = "revenue and share assumed time-matched — un-dated inputs" +NOTE_BASIS_UNSPECIFIED = "shares assumed to be revenue shares" + +# An as-of token: 'YYYY', 'YYYY-MM', or 'YYYY-MM-DD' (ISO order so strings sort). +_AS_OF_RE = re.compile(r"^(\d{4})(?:-(\d{2}))?(?:-(\d{2}))?$") + # The unconditional Q20 label every surface renders/ships, machine-readably: # shares are ALWAYS analyst estimates, so the implied TAM stays modeled even # when every revenue is a sourced filing. @@ -94,6 +111,14 @@ class Player: # figure (whole-company revenue, not in-market) — rendered next to the line # so a sourced-but-total figure is never an unmarked sourced line. revenue_note: str = "" + # When the revenue / the share estimate were measured ('YYYY[-MM[-DD]]'). + # Optional: absent dates only soften the output (assumptions note), present + # dates that disagree past TIME_MATCH_WARN_MONTHS warn — never auto-corrected. + revenue_as_of: str = "" + share_as_of: str = "" + # 'revenue' | 'usage' | '' (unspecified). 'usage' warns: the method's + # denominator must be a revenue share, never seats/usage. + share_basis: str = "" @dataclass(frozen=True, slots=True) @@ -104,6 +129,9 @@ class PlayerImplied: implied_tam: float # revenue / market_share revenue_sourced: bool = False revenue_note: str = "" + revenue_as_of: str = "" + share_as_of: str = "" + share_basis: str = "" @dataclass(frozen=True, slots=True) @@ -178,30 +206,113 @@ def parse_money(raw: str) -> float: raise SenseCheckError(f"could not parse money value {raw!r}") from exc -def split_player_spec(spec: str) -> list[str]: +def parse_as_of(raw: str) -> str: + """Validate an as-of token: ``'2026'``, ``'2026-02'``, or ``'2026-02-15'``. + Returns the token unchanged (ISO order already sorts/compares correctly).""" + s = raw.strip() + m = _AS_OF_RE.match(s) + if not m or (m.group(2) and not 1 <= int(m.group(2)) <= 12): + raise SenseCheckError( + f"could not parse as-of date {raw!r} (use YYYY, YYYY-MM, or YYYY-MM-DD)" + ) + return s + + +def _as_of_month_index(value: str) -> int: + """A validated as-of token as a month count. A bare year is honest about its + imprecision — anchor it mid-year (June) so the comparison stays unbiased.""" + m = _AS_OF_RE.match(value) + assert m is not None # callers validate via parse_as_of / sense_check + return int(m.group(1)) * 12 + (int(m.group(2)) if m.group(2) else 6) + + +def parse_share_basis(raw: str) -> str: + """Validate a share-basis token: ``'revenue'``, ``'usage'``, or empty + (unspecified). Anything else is rejected — never silently coerced.""" + s = raw.strip().lower() + if s and s not in SHARE_BASIS_VALUES: + raise SenseCheckError(f"share_basis must be one of {SHARE_BASIS_VALUES} (got {raw!r})") + return s + + +@dataclass(frozen=True, slots=True) +class PlayerSpec: + """A parsed ``--player`` mini-DSL spec, before money/share token parsing + (the edgar: branch needs the raw revenue token).""" + + name: str + revenue_token: str + share_token: str + sourced: bool = False + revenue_as_of: str = "" + share_as_of: str = "" + share_basis: str = "" + + +# The optional key=value suffix keys the --player DSL accepts (after the three +# positional fields and the optional 'sourced' flag). +_SPEC_KEYS = ("rev_date", "share_date", "share_basis") +_SPEC_SHAPE = "'Name|revenue|share[|sourced][|key=value ...]'" + + +def split_player_spec(spec: str) -> PlayerSpec: """Split a ``--player`` spec into its fields. The field-count and empty-name - guards live HERE only, so every spec branch (plain / edgar) fails the same way.""" + guards live HERE only, so every spec branch (plain / edgar) fails the same way. + + Backward-compatible shape: three positional fields, an optional ``sourced`` + flag, then optional ``key=value`` suffix tokens (``rev_date`` / ``share_date`` + / ``share_basis``) — a token with no ``=`` is only ever the sourced flag.""" parts = [p.strip() for p in spec.split("|")] - if len(parts) not in (3, 4): - raise SenseCheckError(f"--player must be 'Name|revenue|share[|sourced]', got {spec!r}") + if len(parts) < 3: + raise SenseCheckError(f"--player must be {_SPEC_SHAPE}, got {spec!r}") if not parts[0]: raise SenseCheckError(f"player name is empty in {spec!r}") - return parts + sourced = False + kv: dict[str, str] = {} + for i, token in enumerate(parts[3:]): + key, eq, value = token.partition("=") + if eq: + key = key.strip().lower() + if key not in _SPEC_KEYS: + raise SenseCheckError( + f"unknown key {key!r} in {spec!r} (expected one of {_SPEC_KEYS})" + ) + if key in kv: + raise SenseCheckError(f"duplicate key {key!r} in {spec!r}") + kv[key] = value.strip() + elif i == 0 and token.lower() in SOURCED_TOKENS: + sourced = True + else: + raise SenseCheckError(f"--player must be {_SPEC_SHAPE}, got {spec!r}") + return PlayerSpec( + name=parts[0], + revenue_token=parts[1], + share_token=parts[2], + sourced=sourced, + revenue_as_of=parse_as_of(kv["rev_date"]) if kv.get("rev_date") else "", + share_as_of=parse_as_of(kv["share_date"]) if kv.get("share_date") else "", + share_basis=parse_share_basis(kv.get("share_basis", "")), + ) def parse_player(spec: str) -> Player: """Parse a CLI ``--player`` spec ``'Name|revenue|share'`` (optional 4th field - ``sourced`` flags a revenue that comes from a filing). Revenue uses + ``sourced`` flags a revenue that comes from a filing; optional ``key=value`` + suffixes set ``rev_date`` / ``share_date`` / ``share_basis``). Revenue uses :func:`parse_money`; share uses :func:`normalize_share`.""" - parts = split_player_spec(spec) - share_s = parts[2] + ps = split_player_spec(spec) try: - share = normalize_share(float(share_s)) + share = normalize_share(float(ps.share_token)) except ValueError as exc: - raise SenseCheckError(f"could not parse share {share_s!r} in {spec!r}") from exc - sourced = len(parts) == 4 and parts[3].lower() in SOURCED_TOKENS + raise SenseCheckError(f"could not parse share {ps.share_token!r} in {spec!r}") from exc return Player( - name=parts[0], revenue=parse_money(parts[1]), market_share=share, revenue_sourced=sourced + name=ps.name, + revenue=parse_money(ps.revenue_token), + market_share=share, + revenue_sourced=ps.sourced, + revenue_as_of=ps.revenue_as_of, + share_as_of=ps.share_as_of, + share_basis=ps.share_basis, ) @@ -227,8 +338,31 @@ def sense_check( raise SenseCheckError( f"{p.name}: market_share must be in (0, 1] (got {p.market_share})" ) + # Players can be constructed directly (MCP rows, library callers), so + # the as-of / basis tokens are re-validated here, not just at the DSL. + if p.revenue_as_of: + parse_as_of(p.revenue_as_of) + if p.share_as_of: + parse_as_of(p.share_as_of) + parse_share_basis(p.share_basis) warnings: list[str] = [] + for p in players: + # Time-match guardrail: warn, never auto-correct — the analyst owns the + # inputs; the tool's job is to make the mismatch impossible to miss. + if p.revenue_as_of and p.share_as_of: + delta = abs(_as_of_month_index(p.revenue_as_of) - _as_of_month_index(p.share_as_of)) + if delta > TIME_MATCH_WARN_MONTHS: + warnings.append( + f"{p.name}: revenue is as of {p.revenue_as_of} but the share is as of " + f"{p.share_as_of} (~{delta} months apart) — pairing a current revenue " + f"with a stale share inflates/deflates the implied TAM; time-match the inputs" + ) + if p.share_basis == "usage": + warnings.append( + f"{p.name}: share_basis is 'usage' — the denominator must be a REVENUE " + f"share; usage/seat share understates the implied market" + ) total_rev = sum(p.revenue for p in players) total_share = sum(p.market_share for p in players) @@ -272,6 +406,9 @@ def sense_check( p.revenue / p.market_share, p.revenue_sourced, p.revenue_note, + p.revenue_as_of, + p.share_as_of, + p.share_basis, ) for p in players ) @@ -306,6 +443,15 @@ def sense_check( if p.revenue_note == REVENUE_NOTE_FILING_TOTAL_CO ) + # Conditional assumptions: softer than warnings — un-dated or basis-less + # inputs are not errors, but what the implied TAM silently assumes about + # them must be on the record. + assumptions = list(ASSUMPTIONS) + if any(not (p.revenue_as_of and p.share_as_of) for p in players): + assumptions.append(NOTE_UNDATED) + if any(not p.share_basis for p in players): + assumptions.append(NOTE_BASIS_UNSPECIFIED) + sourced_low = sourced_high = sourced_mid = vs_mid = None verdict: str | None = None if sourced_scenarios: @@ -339,4 +485,5 @@ def sense_check( sourced_mid=sourced_mid, vs_mid_ratio=vs_mid, verdict=verdict, + assumptions=tuple(assumptions), ) diff --git a/backend/tests/mcp/test_sense_check_tool.py b/backend/tests/mcp/test_sense_check_tool.py index 42b0719..0bd921c 100644 --- a/backend/tests/mcp/test_sense_check_tool.py +++ b/backend/tests/mcp/test_sense_check_tool.py @@ -149,6 +149,116 @@ def test_bad_player_rows_raise_invalid_input(reg) -> None: assert e2.value.code == CODE_INVALID_INPUT +def test_guardrail_fields_round_trip_and_warn(reg) -> None: + out = sense_check_tool( + reg, + players=[ + { + "name": "Cursor", + "revenue": 2e9, + "market_share": 0.18, + "revenue_as_of": "2026-02", + "share_as_of": "2025-06", + "share_basis": "revenue", + }, + {"name": "Copilot", "revenue": 1e9, "market_share": 0.42, "share_basis": "usage"}, + ], + ) + rows = {p["name"]: p for p in out["per_player"]} + assert rows["Cursor"]["revenue_as_of"] == "2026-02" + assert rows["Cursor"]["share_as_of"] == "2025-06" + assert rows["Cursor"]["share_basis"] == "revenue" + assert rows["Copilot"]["share_basis"] == "usage" + assert any("time-match" in w for w in out["warnings"]) # Cursor's 8-month gap + assert any("understates" in w for w in out["warnings"]) # Copilot's usage share + # Copilot is un-dated; Cursor's basis IS specified but Copilot's dates absent. + assert any("un-dated" in a for a in out["assumptions"]) + + +def test_schema_advertises_guardrail_fields() -> None: + props = TOOLS["sense_check"]["inputSchema"]["properties"]["players"]["items"]["properties"] + assert "revenue_as_of" in props + assert "share_as_of" in props + assert props["share_basis"]["enum"] == ["revenue", "usage"] + + +def test_bad_guardrail_tokens_are_invalid_input(reg) -> None: + with pytest.raises(ToolError) as e1: + sense_check_tool( + reg, + players=[{"name": "A", "revenue": 1e9, "market_share": 0.2, "share_basis": "seats"}], + ) + assert e1.value.code == CODE_INVALID_INPUT + with pytest.raises(ToolError) as e2: + sense_check_tool( + reg, + players=[ + {"name": "A", "revenue": 1e9, "market_share": 0.2, "revenue_as_of": "Feb 2026"} + ], + ) + assert e2.value.code == CODE_INVALID_INPUT + + +def test_edgar_threads_period_end_into_revenue_as_of(reg, monkeypatch: pytest.MonkeyPatch) -> None: + import strata.mcp.sense_check_tool as mod + from strata.sizing.revenue_lookup import ResolvedRevenue + + monkeypatch.setenv("STRATA_MCP_LIVE_CONNECTORS", "1") + + def _fake_resolve(ticker: str, **_kw: object) -> ResolvedRevenue: + return ResolvedRevenue( + ticker=ticker.upper(), + cik="0000000001", + entity_name=f"{ticker} Corp", + revenue=1_000_000_000, + fiscal_year=2026, + concept="Revenues", + source_url="https://www.sec.gov/cgi-bin/browse-edgar?CIK=0000000001", + period_end="2026-01-31", + ) + + monkeypatch.setattr(mod, "resolve_revenue", _fake_resolve) + out = sense_check_tool( + reg, + players=[ + { + "name": "Alpha", + "edgar_ticker": "AAA", + "market_share": 0.10, + "share_as_of": "2025-06", + } + ], + ) + # The filing's own period end arrives with no caller-supplied date, and the + # stale share against it fires the time-match warning. + assert out["per_player"][0]["revenue_as_of"] == "2026-01-31" + assert any("time-match" in w for w in out["warnings"]) + + +def test_edgar_rejects_explicit_revenue_as_of(reg, monkeypatch: pytest.MonkeyPatch) -> None: + import strata.mcp.sense_check_tool as mod + + monkeypatch.setenv("STRATA_MCP_LIVE_CONNECTORS", "1") + + def _boom(*_a: object, **_k: object) -> object: + raise AssertionError("conflict must be rejected before any live lookup") + + monkeypatch.setattr(mod, "resolve_revenue", _boom) + with pytest.raises(ToolError) as exc_info: + sense_check_tool( + reg, + players=[ + { + "name": "Alpha", + "edgar_ticker": "AAA", + "market_share": 0.10, + "revenue_as_of": "2026-02", + } + ], + ) + assert exc_info.value.code == CODE_INVALID_INPUT + + def test_call_tool_dispatch(reg) -> None: out = call_tool( reg, diff --git a/backend/tests/sizing/test_sense_check.py b/backend/tests/sizing/test_sense_check.py index f200f20..e350cd4 100644 --- a/backend/tests/sizing/test_sense_check.py +++ b/backend/tests/sizing/test_sense_check.py @@ -9,10 +9,14 @@ from strata.cli import app from strata.sizing.sense_check import ( + NOTE_BASIS_UNSPECIFIED, + NOTE_UNDATED, Player, SenseCheckError, + parse_as_of, parse_money, parse_player, + parse_share_basis, sense_check, share_token_warning, ) @@ -168,6 +172,114 @@ def test_result_carries_assumptions() -> None: assert any("revenue IN THIS MARKET" in a for a in r.assumptions) +# ── time-match guardrail (cofounder feedback: AI coding tools) ──────────────── +def test_time_mismatch_warning_fires_past_six_months() -> None: + # The real case: Cursor's Feb-2026 ARR over its mid-2025 share (~8 months + # apart) inflated the implied market ~2x — the mismatch must warn. + r = sense_check([Player("Cursor", 2e9, 0.18, revenue_as_of="2026-02", share_as_of="2025-06")]) + assert any("time-match" in w and "Cursor" in w for w in r.warnings) + + +def test_time_match_within_six_months_is_clean() -> None: + r = sense_check([Player("A", 1e9, 0.2, revenue_as_of="2026-02", share_as_of="2025-10")]) + assert not any("time-match" in w for w in r.warnings) + + +def test_year_only_dates_anchor_mid_year() -> None: + # A bare year anchors to June: 2025 (→ Jun-2025) vs Feb-2026 is 8 months. + r = sense_check([Player("A", 1e9, 0.2, revenue_as_of="2026-02", share_as_of="2025")]) + assert any("time-match" in w for w in r.warnings) + + +def test_absent_dates_note_assumed_time_matched() -> None: + r = sense_check([Player("A", 1e9, 0.2)]) + assert NOTE_UNDATED in r.assumptions + # Warnings stay quiet — un-dated inputs soften the output, never shout. + assert not any("time-match" in w for w in r.warnings) + + +def test_fully_dated_players_drop_the_undated_note() -> None: + r = sense_check([Player("A", 1e9, 0.2, revenue_as_of="2026-01", share_as_of="2026-01")]) + assert NOTE_UNDATED not in r.assumptions + + +def test_bad_as_of_tokens_rejected() -> None: + with pytest.raises(SenseCheckError): + parse_as_of("Feb 2026") + with pytest.raises(SenseCheckError): + parse_as_of("2026-13") # month out of range + with pytest.raises(SenseCheckError): + sense_check([Player("A", 1e9, 0.2, revenue_as_of="garbage")]) + assert parse_as_of("2026") == "2026" + assert parse_as_of("2026-02") == "2026-02" + assert parse_as_of("2026-02-15") == "2026-02-15" + + +# ── share-basis guardrail (cofounder feedback: Copilot seat-share) ──────────── +def test_usage_share_basis_fires_strong_warning() -> None: + # Copilot's 42% seat-share is not a revenue share — dividing revenue by it + # understates the implied market. + r = sense_check([Player("Copilot", 1e9, 0.42, share_basis="usage")]) + assert any("REVENUE share" in w and "understates" in w for w in r.warnings) + + +def test_revenue_share_basis_is_clean_and_drops_the_note() -> None: + r = sense_check([Player("A", 1e9, 0.2, share_basis="revenue")]) + assert not any("understates" in w for w in r.warnings) + assert NOTE_BASIS_UNSPECIFIED not in r.assumptions + + +def test_unspecified_basis_notes_revenue_assumption() -> None: + r = sense_check([Player("A", 1e9, 0.2)]) + assert NOTE_BASIS_UNSPECIFIED in r.assumptions + + +def test_bad_share_basis_rejected() -> None: + with pytest.raises(SenseCheckError): + parse_share_basis("seats") + with pytest.raises(SenseCheckError): + sense_check([Player("A", 1e9, 0.2, share_basis="mindshare")]) + + +# ── DSL / CSV round-trips for the guardrail fields ──────────────────────────── +def test_parse_player_kv_suffixes_round_trip() -> None: + p = parse_player("Cursor|2b|0.18|rev_date=2026-02|share_date=2025-06|share_basis=usage") + assert p.revenue == pytest.approx(2e9) + assert p.revenue_as_of == "2026-02" + assert p.share_as_of == "2025-06" + assert p.share_basis == "usage" + assert p.revenue_sourced is False + + +def test_parse_player_sourced_flag_coexists_with_kv() -> None: + p = parse_player("Filer|10b|0.3|sourced|share_basis=revenue|share_date=2026") + assert p.revenue_sourced is True + assert p.share_basis == "revenue" + assert p.share_as_of == "2026" + + +def test_parse_player_old_specs_unchanged() -> None: + # Backward compat: the 3- and 4-field shapes parse exactly as before. + p3 = parse_player("Gong|300m|15") + assert (p3.revenue_as_of, p3.share_as_of, p3.share_basis) == ("", "", "") + p4 = parse_player("Salesforce|10b|0.3|sourced") + assert p4.revenue_sourced is True + assert (p4.revenue_as_of, p4.share_as_of, p4.share_basis) == ("", "", "") + + +def test_parse_player_rejects_bad_kv() -> None: + with pytest.raises(SenseCheckError): + parse_player("A|1b|0.1|frobnicate=2026") # unknown key + with pytest.raises(SenseCheckError): + parse_player("A|1b|0.1|rev_date=2026|rev_date=2025") # duplicate key + with pytest.raises(SenseCheckError): + parse_player("A|1b|0.1|share_basis=seats") # bad basis + with pytest.raises(SenseCheckError): + parse_player("A|1b|0.1|rev_date=Feb 2026") # bad date + with pytest.raises(SenseCheckError): + parse_player("A|1b|0.1|sourced|bogus") # bare token past position 4 + + # ── CLI ─────────────────────────────────────────────────────────────────────── def test_cli_sense_check_standalone() -> None: r = runner.invoke( @@ -310,3 +422,74 @@ def test_cli_sense_check_against_curated_concept() -> None: assert r.exit_code == 0, r.stdout assert "cross-check vs demo-ai-sales-ops-us" in r.stdout assert "verdict:" in r.stdout + + +def test_cli_time_mismatch_and_usage_basis_warn() -> None: + r = runner.invoke( + app, + [ + "sense-check", + "--player", + "Cursor|2b|0.18|rev_date=2026-02|share_date=2025-06", + "--player", + "Copilot|1b|0.42|share_basis=usage", + ], + ) + assert r.exit_code == 0, r.stdout + assert "time-match" in r.stdout # 8-month gap fires the warning + assert "understates" in r.stdout # usage-basis strong warning + assert NOTE_UNDATED in r.stdout # Copilot is un-dated → assumptions note + assert NOTE_BASIS_UNSPECIFIED in r.stdout # Cursor's basis unspecified + + +def test_cli_edgar_period_end_threads_into_revenue_as_of( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The filing already knows WHEN its revenue is measured — a stale share date + # against it must fire the time-match warning with no rev_date typed. + import strata.cli as cli + from strata.sizing.revenue_lookup import ResolvedRevenue + + def _fake_resolve(ticker: str, **_kw: object) -> ResolvedRevenue: + return ResolvedRevenue( + ticker=ticker.upper(), + cik="0000000001", + entity_name="Cursor Listed Twin Inc.", + revenue=2_000_000_000, + fiscal_year=2026, + concept="Revenues", + source_url="https://www.sec.gov/cgi-bin/browse-edgar?CIK=0000000001", + period_end="2026-01-31", + ) + + monkeypatch.setattr(cli, "resolve_revenue", _fake_resolve) + r = runner.invoke(app, ["sense-check", "--player", "Cursor|edgar:CRSR|0.18|share_date=2025-06"]) + assert r.exit_code == 0, r.stdout + assert "2026-01-31" in r.stdout and "time-match" in r.stdout + + +def test_cli_edgar_rejects_explicit_rev_date(monkeypatch: pytest.MonkeyPatch) -> None: + import strata.cli as cli + + def _boom(*_a: object, **_k: object) -> object: + raise AssertionError("conflict must be rejected before any live lookup") + + monkeypatch.setattr(cli, "resolve_revenue", _boom) + r = runner.invoke(app, ["sense-check", "--player", "A|edgar:CRM|0.3|rev_date=2026-02"]) + assert r.exit_code == 2 + assert "fiscal-period end" in r.output + + +def test_cli_csv_guardrail_columns_round_trip(tmp_path: Path) -> None: + csv_path = tmp_path / "players.csv" + csv_path.write_text( + "name,revenue,market_share,sourced,revenue_as_of,share_as_of,share_basis\n" + "Cursor,2b,18,,2026-02,2025-06,revenue\n" + "Copilot,1b,42,,,,usage\n", + encoding="utf-8", + ) + r = runner.invoke(app, ["sense-check", "--players-file", str(csv_path)]) + assert r.exit_code == 0, r.stdout + assert "time-match" in r.stdout # Cursor's 8-month gap + assert "understates" in r.stdout # Copilot's usage basis + assert NOTE_UNDATED in r.stdout # Copilot has no dates diff --git a/docs/methods/sense-check-defensibility.md b/docs/methods/sense-check-defensibility.md index 59b4d86..5e7c98d 100644 --- a/docs/methods/sense-check-defensibility.md +++ b/docs/methods/sense-check-defensibility.md @@ -33,6 +33,21 @@ The method: | 9 | **"Private-company revenue is itself a guess — this is circular."** | Each revenue carries a `revenue_sourced` flag; the output shows `n/N revenues sourced` and tags estimates `(est.)`. The **MODELED label is unconditional**: market shares are always analyst estimates, so the implied TAM is labelled **MODELED (Q20)** even when every revenue is a sourced filing, and is **never persisted as a sourced primitive** — it can only agree or disagree with the sourced number, never become one. | | 10 | **"This market isn't revenue-defined yet"** — pre-revenue / latent demand. | Assumption #4 ("the market is revenue-defined today, not pre-revenue / latent"). For a `latent_demand` market the method is the wrong tool, and the assumption says so — the reviewer uses the adoption-curve lens instead. | +## Two more failure modes — validated in practice (AI coding tools) + +Both surfaced when the method was applied for real to the AI-coding-tools market +(prod MCP feedback, feedback event #6). Both are **warnings, never silent +corrections** — the tool stays honest, not clever. + +| # | Failure mode | How the check answers it | +|---|---|---| +| 11 | **"Your revenue and your share aren't from the same moment."** Dividing Cursor's Feb-2026 $2B ARR by its mid-2025 ~18% share inflates the implied market roughly 2× — the share was measured before the revenue grew into it. | Optional `revenue_as_of` / `share_as_of` dates (`YYYY[-MM[-DD]]`) on every player, via the `--player` DSL (`rev_date=2026-02\|share_date=2025-06`), the CSV columns, and the MCP row schema. When both dates are present and more than ~6 months apart (`TIME_MATCH_WARN_MONTHS`), a per-player warning fires ("pairing a current revenue with a stale share inflates/deflates the implied TAM; time-match the inputs"). When dates are absent, the assumptions block adds the softer line "revenue and share assumed time-matched — un-dated inputs". An `edgar:` revenue threads its filing's own fiscal-period end into `revenue_as_of` automatically — and rejects a conflicting caller-supplied date rather than silently overriding either. A bare year anchors to mid-year (June), staying honest about its own imprecision. | +| 12 | **"That's a seat share, not a revenue share."** GitHub Copilot's ~42% share of AI-coding *seats* is far above its revenue share; dividing market revenue by a usage/seat share understates the implied market. | Optional `share_basis` per player: `revenue` \| `usage` \| unspecified (DSL `share_basis=usage`, CSV column, MCP enum). `usage` fires a strong per-player warning ("denominator must be a REVENUE share — usage/seat share understates the implied market"); unspecified adds the assumptions line "shares assumed to be revenue shares". Any other token is rejected, never coerced. | + +These sharpen arguments #2 and #7 above from stated assumptions into checkable, +per-player guardrails — when the analyst supplies the metadata, the tool checks +it; when they don't, the output says exactly what was assumed. + ### Bonus: "comparing to one number hides the real spread" When `--concept` is given, the implied TAM is compared to the curated market's **full sourced scenario range** (`within / above / below`), not a single point —