From bda774052b03795b2b995dafd8ba7b94730a729e Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 15:20:57 +0200 Subject: [PATCH 1/3] fix(engine): Accept host token counts and keep base64 media out of the ratio Size gains Tokens (used as given, not Estimated) and MediaBytes. PosHash gains Media, set by NewBody for base64 image and document blocks. The bytes-per-token ratio now covers text only; media is priced at 1 token per 250 bytes. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- docs/design.md | 17 +++-- docs/spec.md | 6 +- internal/engine/body.go | 44 ++++++++++++- internal/engine/compute.go | 46 +++++++++----- internal/engine/doc.go | 8 ++- internal/engine/fingerprint.go | 44 ++++++++++++- internal/engine/size_tokens_test.go | 96 +++++++++++++++++++++++++++++ internal/engine/types.go | 17 ++++- 8 files changed, 247 insertions(+), 31 deletions(-) create mode 100644 internal/engine/size_tokens_test.go diff --git a/docs/design.md b/docs/design.md index 6dced2c..6e8ba38 100644 --- a/docs/design.md +++ b/docs/design.md @@ -271,6 +271,14 @@ and can be reported as unpriced ([R19](spec.md#r19)). cache write). The provider bills the forwarded body, so the ratio is taken on it. If the request has no usage, the library uses a fixed fallback ratio. The library marks every priced result that uses this conversion as `Estimated`. No operator names a tokenizer, so two operators give the same tokens for the same bytes. + Two exceptions keep the ratio true (accountant#11): + - **Base64 media stays out of the ratio.** `NewBody` counts the base64 payload of image and + document blocks in `PosHash.Media`. The ratio uses the text bytes only, over the billed tokens + less the media tokens. Media is priced at a fixed 1 token for 250 bytes. A host that builds a + `Body` itself sets `PosHash.Media`. A host marks media inside a `Size` with `Size.MediaBytes`. + - **The host may pass tokens.** If a host knows the tokens of a range (a tokenizer, or a + count_tokens call), it sets `Size.Tokens`. The library uses them as they are, and does not mark + that size `Estimated`. `Tokens` 0 means not measured, and the bytes are converted. 8. **The engine reports a change that `apply` writes, under the context operator that decided it.** `toolschema`, `toolfilter` and `cachesplit` are marker components: their `Reformat()` does nothing, and `apply` performs the rewrite (`apply/envelope.go`, `apply/prefixsplit.go`). `apply` @@ -787,8 +795,8 @@ cache break (§6.2), which counts forwarded content between the break and B\*. T Both hosts send bytes for each forwarded position (`Body`) and for each operation (`Size`). The library converts both with the request's own ratio (billed prompt tokens ÷ body bytes), as `pricing.EstimateTokensFromBytes` does today. Results that use the ratio are marked `Estimated`. -No host sends a token count, so the same tokens in both hosts holds whenever the two hosts receive -the same bodies and the same operations. +A host may also send a token count in `Size.Tokens` (§4.2 rule 7). The same tokens in both hosts +holds whenever the two hosts receive the same bodies and the same operations. ## 6. The library @@ -1225,8 +1233,9 @@ Where a question has test cases, a host that compares them with its own code mar - **Q3. Accuracy of the bytes ratio.** The request ratio is an average. Removed content (large JSON tool output) may have fewer bytes per token than the average request. Option: `Size` takes a content class (`text`, `json`, `binary`) and the library keeps a ratio for each class. Decide - after measuring the error on real sessions. An optional `Tokens` field on `Size`, which - the library would use instead of estimating, is also possible. + after measuring the error on real sessions. The optional `Tokens` field on `Size` is done + (accountant#11), as is the exclusion of base64 media from the ratio. A content class and an error + range for each `PricedOp` are still open. - **Q4. Where the previous request's hashes live.** The counterfactual needs the previous request's hashes on the same thread. The current design (D12, wiring document) has the host keep one diff --git a/docs/spec.md b/docs/spec.md index af5c5bb..69dd3de 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -280,8 +280,10 @@ numbers are not used: those requirements were dropped. ### C. Facts for pricing - **R7.** Each context operation has a position: section and index, in the provider's hash order. -- **R8.** Sizes are in bytes. A context operator sends no tokens. The library converts bytes to - tokens and marks the result `Estimated` ([§4.2](design.md#42-rules) rule 7). +- **R8.** Sizes are in bytes. The library converts bytes to tokens and marks the result `Estimated` + ([§4.2](design.md#42-rules) rule 7). A host that knows the tokens of a range may send them in `Size.Tokens`. The + library then uses them and does not mark that size `Estimated`. Base64 image and document bytes are not in the + conversion ratio. - **R11.** Model-call usage is the provider's billed usage, read from the call's own response by the model client the host gives the engine. The engine does not count it (wiring document). - **R22.** A change has an ID that is the same on every request that carries it (`ChangeID`). diff --git a/internal/engine/body.go b/internal/engine/body.go index 7d42635..19ea80e 100644 --- a/internal/engine/body.go +++ b/internal/engine/body.go @@ -103,16 +103,54 @@ func lastAtOrBefore(bps []bp, x int) int { return r } -// ratio converts forwarded bytes to tokens with the request's own ratio (§5.3). +// mediaBytesPerToken is the fixed price of a media block (a base64 image or document): about one +// token for each 250 bytes of its base64 text. Media tokens do not follow the text ratio, so the +// request ratio leaves the media bytes out and takes the media tokens out of the billed total +// first (§4.2 rule 7). +const mediaBytesPerToken = 250 + +// cumMedia returns the media bytes of positions 0..i; 0 for i < 0. +func (b Body) cumMedia(i int) int { + n := 0 + for j := 0; j <= i && j < len(b.Pos); j++ { + n += int(b.Pos[j].Media) + } + return n +} + +// ratio converts forwarded text bytes to tokens with the request's own ratio (§5.3). The bytes and +// the tokens are those of the text only: media is priced apart (mediaBytesPerToken). type ratio struct{ tokens, bytes int } +// newRatio builds the ratio of a body that the provider billed for billed prompt tokens. +func newRatio(billed int, b Body) ratio { + last := len(b.Pos) - 1 + media := b.cumMedia(last) + return ratio{tokens: max(billed-media/mediaBytesPerToken, 0), bytes: b.cumBytes(last) - media} +} + func (r ratio) of(bytes int) int { - if r.bytes == 0 { + if r.bytes <= 0 { return 0 } return int(int64(bytes) * int64(r.tokens) / int64(r.bytes)) } +// tokensOf converts bytes of which media are media bytes: the text with the ratio, the media with +// the fixed price. +func (r ratio) tokensOf(bytes, media int) int { + return r.of(bytes-media) + media/mediaBytesPerToken +} + +// upTo is the tokens of positions 0..i of b. +func (r ratio) upTo(b Body, i int) int { return r.tokensOf(b.cumBytes(i), b.cumMedia(i)) } + +// between is the tokens of positions from+1..to of b (the bytes of position from are not counted +// twice: the same difference as cumBytes(to) - cumBytes(from)). +func (r ratio) between(b Body, from, to int) int { + return r.tokensOf(b.cumBytes(to)-b.cumBytes(from), b.cumMedia(to)-b.cumMedia(from)) +} + // extent returns the last candidate position whose estimated prefix fits in tokens, or -1. The // tolerance absorbs the error of the byte ratio and the 128-token rounding of older OpenAI models. // With no candidate breakpoint every position is a candidate. @@ -123,7 +161,7 @@ func extent(b Body, r ratio, tokens int, candidates []bp) int { limit := tokens + 128 + tokens/50 best := -1 try := func(i int) { - if i > best && r.of(b.cumBytes(i)) <= limit { + if i > best && r.upTo(b, i) <= limit { best = i } } diff --git a/internal/engine/compute.go b/internal/engine/compute.go index a44f8c7..7c3ec4e 100644 --- a/internal/engine/compute.go +++ b/internal/engine/compute.go @@ -192,11 +192,11 @@ func isChange(op Operation) bool { } func negative(op Operation) bool { - if op.SizeBefore.Bytes < 0 || op.SizeAfter.Bytes < 0 { + if op.SizeBefore.invalid() || op.SizeAfter.invalid() { return true } for _, s := range op.Spans { - if s.SizeBefore.Bytes < 0 || s.SizeAfter.Bytes < 0 { + if s.SizeBefore.invalid() || s.SizeAfter.invalid() { return true } } @@ -230,6 +230,16 @@ func apply(p Position, s *MessageSplice, into bool) (Position, bool) { return p, into } +// sizeTokens converts one reported size to tokens. A size the host counted (Tokens > 0) is exact. +// Any other size is converted from its bytes with the request ratio, and so is an estimate. A size +// of zero bytes is exactly zero tokens (§4.2 rule 7). +func (c *calcState) sizeTokens(s Size) (tokens int, exact bool) { + if s.Tokens > 0 { + return s.Tokens, true + } + return c.ratio.tokensOf(s.Bytes, s.MediaBytes), s.Bytes == 0 +} + // calcState is the working state of one calculation. type calcState struct { r *Request @@ -264,7 +274,7 @@ func (r *Request) newCalc(s ThreadState, items []*item, usage Usage) *calcState c := &calcState{r: r, s: s, items: items, usage: usage, client: r.Client, fwd: r.Forwarded, prevExt: -1, bstar: -1, bstarFwd: -1, readEnd: -1, cachedFwd: -1, prevExtCli: -1} c.noUsage = usage == (Usage{}) - c.ratio = ratio{tokens: usage.Input + usage.CacheRead + usage.CacheWrite, bytes: r.Forwarded.cumBytes(len(r.Forwarded.Pos) - 1)} + c.ratio = newRatio(usage.Input+usage.CacheRead+usage.CacheWrite, r.Forwarded) c.clientBPs = c.client.breakpoints(r.ClientBreakpoints) c.fwdBPs = c.fwd.breakpoints(r.Breakpoints) if p := s.PrevPrint; p != nil { @@ -459,7 +469,7 @@ func (c *calcState) pingRead() *Position { // Only the tokens the ping cached are in that body. Its Input after the prefix (a sweep question, // a summary request) is billed but is not part of the body, so it would raise the ratio and pull // the extent back to an earlier message. - r := ratio{tokens: u.CacheRead + u.CacheWrite, bytes: prev.Forwarded.cumBytes(len(prev.Forwarded.Pos) - 1)} + r := newRatio(u.CacheRead+u.CacheWrite, prev.Forwarded) i := -1 if r.tokens > 0 { // a usage that bills no prompt token proves nothing about the extent i = extent(prev.Forwarded, r, u.CacheRead+u.CacheWrite, nil) @@ -554,7 +564,7 @@ func (c *calcState) phaseAndBoundary(res *Result) { // readCoversExtent reports whether the tokens the provider read reach the previous request's cached extent, // with the tolerance of extent. The extent is priced at the byte ratio of this request. func (c *calcState) readCoversExtent(prev *Fingerprint) bool { - want := c.ratio.of(prev.Forwarded.cumBytes(c.prevExt)) + want := c.ratio.upTo(prev.Forwarded, c.prevExt) return want > 0 && want <= c.usage.CacheRead+128+c.usage.CacheRead/50 } @@ -620,7 +630,7 @@ func (c *calcState) cacheEffects(res *Result) { // past the 5m TTL only the entries held at 1h are still alive: a 5m part past the head expired expect = min(expect, c.headEnd()) } - predicted := c.ratio.of(c.fwd.cumBytes(expect)) + predicted := c.ratio.upTo(c.fwd, expect) low := c.usage.CacheRead+128+predicted/50 < predicted // the one-block tolerance (row 4) if !low { return @@ -697,7 +707,7 @@ func (c *calcState) addBreak(res *Result, from int, changeID string) { if c.bstarFwd >= 0 { p := c.fwd.position(c.bstarFwd) e.To = &p - if n := c.ratio.of(c.fwd.cumBytes(c.bstarFwd) - c.fwd.cumBytes(from)); n > 0 { + if n := c.ratio.between(c.fwd, from, c.bstarFwd); n > 0 { e.Saved.CacheRead = n addWrite(&e.Caused, ttlAtOrAfter(c.fwdBPs, c.bstarFwd), n) // "" counts as 5m } @@ -717,7 +727,7 @@ func (c *calcState) rescue(res *Result) { if c.usage.CacheRead <= 0 { return } - n := min(c.ratio.of(c.fwd.cumBytes(mustFlat(c.fwd, *c.pingRead()))), c.usage.CacheRead) + n := min(c.ratio.upTo(c.fwd, mustFlat(c.fwd, *c.pingRead())), c.usage.CacheRead) if w := c.r.Facts.SharedWarm; w != nil { i, ok := c.fwd.flat(*w) if !ok { // a host bug: a doubtful figure may only lower a saving, so credit nothing (R54) @@ -727,7 +737,7 @@ func (c *calcState) rescue(res *Result) { } // a sibling kept this part warm without the ping (R54). Two floors, so n can be 1 token off the single // conversion: the same tolerance the conservation check allows. - n -= c.ratio.of(c.fwd.cumBytes(i)) + n -= c.ratio.upTo(c.fwd, i) } if n <= 0 { return @@ -763,7 +773,7 @@ func (c *calcState) headTTL(res *Result) { if head < 0 { return } - n := min(c.ratio.of(c.fwd.cumBytes(head)), c.usage.CacheRead) + n := min(c.ratio.upTo(c.fwd, head), c.usage.CacheRead) e := CacheEffect{Kind: EffectTTLRescue, ChangeID: id, TTL: "1h", Estimated: true} e.Saved.CacheWrite5m, e.Caused.CacheRead = n, n c.addEffect(res, e) @@ -791,9 +801,9 @@ func (c *calcState) split(res *Result) { if c.s.PrevPrint == nil || c.clientFD >= c.client.sectionEnd(1) { return } - n := min(c.ratio.of(c.fwd.cumBytes(c.fwdFD-1)), c.usage.CacheRead) + n := min(c.ratio.upTo(c.fwd, c.fwdFD-1), c.usage.CacheRead) if c.bstarFwd >= 0 { // the client body would have read up to B* anyway: only the rest is the split's - n -= c.ratio.of(c.fwd.cumBytes(c.bstarFwd)) + n -= c.ratio.upTo(c.fwd, c.bstarFwd) } if n <= 0 { return @@ -871,6 +881,7 @@ func (c *calcState) price(res *Result) { continue } p := priced(it.op, it.id) + estimated := false // true when a size is converted from bytes (§4.2 rule 7) switch { case it.bad != "": res.Unpriced = append(res.Unpriced, Unpriced{OpID: it.id, Reason: it.bad}) @@ -879,9 +890,13 @@ func (c *calcState) price(res *Result) { case resultUnknown(it.op): res.Unpriced = append(res.Unpriced, Unpriced{OpID: it.id, Reason: ReasonPingResultUnknown}) case it.op.Kind == KindRewrite: - p.Estimated = true // the tokens come from the bytes ratio (§4.2 rule 7) for _, sp := range it.spans { - d := c.ratio.of(sp.before.Bytes) - c.ratio.of(sp.after.Bytes) + bt, bx := c.sizeTokens(sp.before) + at, ax := c.sizeTokens(sp.after) + if !bx || !ax { + estimated = true + } + d := bt - at if d > 0 { addTier(&p.Saved, mustFlat(c.client, sp.from), c.bstar, c.clientBPs, d) } else if d < 0 { @@ -900,6 +915,7 @@ func (c *calcState) price(res *Result) { } } } + p.Estimated = p.Estimated || estimated if it.active { active = append(active, p) } else { @@ -926,7 +942,7 @@ func (c *calcState) conservation(res *Result) { tol++ } billed := c.usage.Input + c.usage.CacheRead + c.usage.CacheWrite - want := c.ratio.of(c.client.cumBytes(len(c.client.Pos)-1)) - billed + want := c.ratio.upTo(c.client, len(c.client.Pos)-1) - billed if d := want - net; d > tol+1 || d < -(tol+1) { res.Checks = append(res.Checks, CheckFailure{Check: "conservation", Detail: fmt.Sprintf("bill difference=%d ops=%d tokens", want, net)}) diff --git a/internal/engine/doc.go b/internal/engine/doc.go index 76a9405..9d87c79 100644 --- a/internal/engine/doc.go +++ b/internal/engine/doc.go @@ -12,10 +12,12 @@ // The design leaves some rules open. This implementation decides them as follows. Each one is // the simplest rule that the design allows. // -// 1. Tokens from bytes. Size is bytes only (§4.2 rule 7). The ratio is the billed prompt tokens +// 1. Tokens from bytes. A Size is bytes (§4.2 rule 7), or tokens when the host sets Size.Tokens: +// those are used as they are and are not Estimated. The ratio is the billed prompt tokens // (input + cache read + cache write) divided by the bytes of the forwarded body, because the -// billed tokens bill the forwarded body (§5.3). Every priced rewrite and every effect that uses -// the ratio is Estimated. +// billed tokens bill the forwarded body (§5.3). Base64 media (PosHash.Media, Size.MediaBytes) +// is left out of the ratio and priced at a fixed 250 bytes a token. Every priced rewrite that +// converts bytes and every effect that uses the ratio is Estimated. // 2. No usage. A response with no usage prices nothing (§8): every operation goes to Unpriced // with ReasonNoUsage. The fixed fallback ratio of §4.2 rule 7 is not used. // 3. Extents from usage. The provider reports token counts, not positions. The read extent is the diff --git a/internal/engine/fingerprint.go b/internal/engine/fingerprint.go index 57a5153..548a1c2 100644 --- a/internal/engine/fingerprint.go +++ b/internal/engine/fingerprint.go @@ -1,6 +1,9 @@ package engine -import "time" +import ( + "bytes" + "time" +) // Fingerprint is the most recent request of one thread (§5.1). The library builds it inside // Compute, because Cached comes from the usage, which arrives after the response. @@ -29,6 +32,9 @@ type Body struct { type PosHash struct { Hash uint64 `json:"hash"` // cumulative: equal hashes mean an equal prefix up to this position Bytes int32 `json:"bytes"` + // Media is the part of Bytes that is a base64 image or document block. NewBody finds it. The + // token ratio leaves it out (§4.2 rule 7). It is not in the hash. + Media int32 `json:"media,omitempty"` } // Breakpoint is a cache breakpoint in a body. @@ -49,8 +55,42 @@ func NewBody(tools, system, messages [][]byte) Body { h = (h ^ uint64(c)) * 1099511628211 } h = (h ^ 0xff) * 1099511628211 // position separator - b.Pos = append(b.Pos, PosHash{Hash: h, Bytes: int32(len(p))}) + b.Pos = append(b.Pos, PosHash{Hash: h, Bytes: int32(len(p)), Media: int32(mediaBytes(p))}) } } return b } + +// mediaBytes counts the base64 payload of the media blocks in one position: the value of each +// "data" string that follows a "base64" source type, as in {"type":"base64","media_type":"image/png", +// "data":"..."}. It does not parse JSON, and the payload is never kept (R2). +func mediaBytes(p []byte) int { + const marker = `"base64"` + n := 0 + for rest := p; ; { + i := bytes.Index(rest, []byte(marker)) + if i < 0 { + return n + } + rest = rest[i+len(marker):] + j := bytes.Index(rest, []byte(`"data"`)) + if j < 0 { + return n + } + k := j + len(`"data"`) + // skip spaces and the colon, then the opening quote + for k < len(rest) && (rest[k] == ' ' || rest[k] == ':') { + k++ + } + if k >= len(rest) || rest[k] != '"' { + continue + } + k++ + end := bytes.IndexByte(rest[k:], '"') + if end < 0 { + return n + len(rest) - k + } + n += end + rest = rest[k+end:] + } +} diff --git a/internal/engine/size_tokens_test.go b/internal/engine/size_tokens_test.go new file mode 100644 index 0000000..59d8aa3 --- /dev/null +++ b/internal/engine/size_tokens_test.go @@ -0,0 +1,96 @@ +package engine + +import ( + "strings" + "testing" +) + +// Issue 11: one bytes-per-token ratio misprices images and dense JSON. + +// wrapper is the JSON text around the payload of imageBlock: text, 4 bytes a token here. +var wrapper = len(imageBlock(0)) + +// imageBlock is a message that holds a base64 image of n payload bytes. +func imageBlock(n int) []byte { + return []byte(`{"type":"image","source":{"type":"base64","media_type":"image/png","data":"` + + strings.Repeat("A", n) + `"}}`) +} + +// oneOp runs one request with one operation and returns the priced operation. +func oneOp(t *testing.T, body Body, op Operation, usage Usage) (Result, PricedOp) { + t.Helper() + ts := NewTrackers() + r := ts.BeginRequest(RequestMeta{RequestID: "r", Session: "s", At: t0}, body, nil) + r.Add(op) + r.EndRequest(Thread{ID: "main"}, body, nil, exact5m) + res := r.Compute(usage) + return res, findOp(t, res, "o1") +} + +func TestNewBodyCountsBase64MediaApart(t *testing.T) { + img := imageBlock(400000) + body := NewBody(nil, nil, [][]byte{img, []byte("plain text")}) + if got := body.Pos[0].Media; got != 400000 { + t.Fatalf("Media = %d, want 400000", got) + } + if body.Pos[1].Media != 0 { + t.Fatalf("text position Media = %d, want 0", body.Pos[1].Media) + } +} + +// A 400,000-byte image (1600 tokens) and 3 text messages of 4000 bytes (1000 tokens each). +// A cut of 4000 text bytes saves 1000 tokens. Before the fix the library booked 54. +func TestImageDoesNotSkewTheTextRatio(t *testing.T) { + text := func(id string) []byte { return []byte(id + strings.Repeat(".", 4000-len(id))) } + body := NewBody(nil, nil, [][]byte{imageBlock(400000), text("a"), text("b"), text("c")}) + op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(1), + SizeBefore: Size{Bytes: 4000}} + _, p := oneOp(t, body, op, Usage{Input: 1600 + (3*4000+wrapper)/4}) + if got := tierSum(p.Saved); got < 998 || got > 1000 { // the wrapper text rounds + t.Fatalf("saved = %d tokens, want 1000", got) + } + if !p.Estimated { + t.Fatal("a bytes conversion must stay Estimated") + } +} + +// 10,000 bytes of dense JSON are 5000 tokens. The request ratio would book 3500. +func TestHostTokensAreUsedAsTheyAre(t *testing.T) { + json := []byte("j" + strings.Repeat("{", 9999)) + prose := []byte("p" + strings.Repeat(".", 39999)) + body := NewBody(nil, nil, [][]byte{json, prose}) + op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(0), + SizeBefore: Size{Bytes: 10000, Tokens: 5000}} + _, p := oneOp(t, body, op, Usage{Input: 15000}) + if got := tierSum(p.Saved); got != 5000 { + t.Fatalf("saved = %d, want the host's 5000", got) + } + if p.Estimated { + t.Fatal("host-counted tokens must not be Estimated") + } +} + +func TestMediaBytesOnASizeUseTheFixedMediaPrice(t *testing.T) { + img := imageBlock(400000) + body := NewBody(nil, nil, [][]byte{img, []byte("t" + strings.Repeat(".", 3999))}) + op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(0), + SizeBefore: Size{Bytes: len(img), MediaBytes: 400000}} + _, p := oneOp(t, body, op, Usage{Input: 1600 + (4000+wrapper)/4}) + want := 1600 + wrapper/4 // the media at the fixed price, the JSON wrapper at the text ratio + if got := tierSum(p.Saved); got < want-2 || got > want { + t.Fatalf("saved = %d, want about %d", got, want) + } +} + +func TestSizeWithMoreMediaThanBytesIsRefused(t *testing.T) { + body := NewBody(nil, nil, [][]byte{[]byte("abc")}) + op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(0), + SizeBefore: Size{Bytes: 10, MediaBytes: 11}} + ts := NewTrackers() + r := ts.BeginRequest(RequestMeta{RequestID: "r", Session: "s", At: t0}, body, nil) + r.Add(op) + r.EndRequest(Thread{ID: "main"}, body, nil, exact5m) + if !hasUnpriced(r.Compute(Usage{Input: 3}), ReasonNegativeSize) { + t.Fatal("want negative_size") + } +} diff --git a/internal/engine/types.go b/internal/engine/types.go index 4ad323d..8d10342 100644 --- a/internal/engine/types.go +++ b/internal/engine/types.go @@ -120,10 +120,23 @@ type SettingChange struct { At *Position `json:"at,omitempty"` } -// Size is bytes only. A context operator does not count tokens: the library converts bytes to -// tokens (§4.2 rule 7). +// Size is the bytes of a range, and optionally its tokens. Bytes are required. When the host knows +// the tokens (a tokenizer, or a count_tokens call), it sets Tokens, and the library uses them as +// they are. Without Tokens the library converts the bytes (§4.2 rule 7). type Size struct { Bytes int `json:"bytes,omitempty"` // required on rewrite; the sum check compares it with the Body sizes (§6.5) + // Tokens is the token count the host measured for the range. 0 means not measured: the library + // then converts Bytes and marks the result Estimated. + Tokens int `json:"tokens,omitempty"` + // MediaBytes is the part of Bytes that is base64 media (an image or a document block). The + // library prices it apart from the text ratio. 0 means the range has no media, or the host did + // not mark it. + MediaBytes int `json:"mediaBytes,omitempty"` +} + +// invalid says whether a size cannot be real: a negative figure, or more media than bytes. +func (s Size) invalid() bool { + return s.Bytes < 0 || s.Tokens < 0 || s.MediaBytes < 0 || s.MediaBytes > s.Bytes } // MessageSplice describes a change to the message list. From 8ebda2ab02ca7f6c4175b137c280766491849c22 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 15:34:33 +0200 Subject: [PATCH 2/3] fix(engine): Parse media blocks, keep conservation exact with host tokens mediaBytes now parses JSON and counts only base64 image/document source data and base64 data URLs. Check 6 compares host-token sizes at the ratio. Media above Bytes is capped, and a bill below the media price fails the media_tokens check. Docs list Size.Tokens, MediaBytes and PosHash.Media. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- docs/design.md | 18 ++-- internal/engine/body.go | 3 +- internal/engine/compute.go | 11 +++ internal/engine/fingerprint.go | 119 +++++++++++++++++++++----- internal/engine/size_tokens_test.go | 127 ++++++++++++++++++++++------ 5 files changed, 220 insertions(+), 58 deletions(-) diff --git a/docs/design.md b/docs/design.md index 6e8ba38..b8c0e42 100644 --- a/docs/design.md +++ b/docs/design.md @@ -178,9 +178,11 @@ type SettingChange struct { At *Position } -// Size is bytes only. A context operator does not count tokens: the library converts bytes to tokens (§4.2 rule 7). +// Size is the bytes of a range, and optionally its tokens (§4.2 rule 7). type Size struct { - Bytes int // required on rewrite; the sum check compares it with the Body sizes (§6.5) + Bytes int // required on rewrite; the sum check compares it with the Body sizes (§6.5) + Tokens int // the host's count; 0 = not measured, so the library converts Bytes and marks it Estimated + MediaBytes int // the base64 part of Bytes; priced at a fixed 250 bytes a token, apart from the text ratio } type MessageSplice struct{ At, Removed, Inserted int } // messages only @@ -272,9 +274,10 @@ and can be reported as unpriced ([R19](spec.md#r19)). marks every priced result that uses this conversion as `Estimated`. No operator names a tokenizer, so two operators give the same tokens for the same bytes. Two exceptions keep the ratio true (accountant#11): - - **Base64 media stays out of the ratio.** `NewBody` counts the base64 payload of image and - document blocks in `PosHash.Media`. The ratio uses the text bytes only, over the billed tokens - less the media tokens. Media is priced at a fixed 1 token for 250 bytes. A host that builds a + - **Base64 media stays out of the ratio.** `NewBody` parses each position and counts the `data` + of an image or document block whose source has `type: base64`, and the payload of a `data:...;base64,` URL + (OpenAI form), in `PosHash.Media`. Text that only quotes "base64" is not media. `Media` above `Bytes` is capped. The ratio uses the text bytes only, over the billed tokens + less the media tokens (if the bill is below the media price, check `media_tokens` fails). Media is priced at a fixed 1 token for 250 bytes. A host that builds a `Body` itself sets `PosHash.Media`. A host marks media inside a `Size` with `Size.MediaBytes`. - **The host may pass tokens.** If a host knows the tokens of a range (a tokenizer, or a count_tokens call), it sets `Size.Tokens`. The library uses them as they are, and does not mark @@ -674,6 +677,7 @@ type Body struct { type PosHash struct { Hash uint64 // cumulative: equal hashes mean an equal prefix up to this position Bytes int32 + Media int32 // the base64 image/document part of Bytes (Anthropic source blocks and data URLs); not in the hash } ``` @@ -688,7 +692,7 @@ The fingerprint itself is built inside `Compute`. `Cached` comes from the usage, after the response, so no earlier call can build it. `Compute` keeps the result in `ThreadState.PrevPrint` for the next request. -`PosHash` is 16 bytes in memory (8 for the hash, 4 for the size, 4 of padding). It holds hashes and sizes only ([R2](spec.md#r2)). +`PosHash` is 16 bytes in memory (8 for the hash, 4 for the size, 4 for the media size). It holds hashes and sizes only ([R2](spec.md#r2)). ### 5.2 Classifying a break @@ -796,7 +800,7 @@ Both hosts send bytes for each forwarded position (`Body`) and for each operatio library converts both with the request's own ratio (billed prompt tokens ÷ body bytes), as `pricing.EstimateTokensFromBytes` does today. Results that use the ratio are marked `Estimated`. A host may also send a token count in `Size.Tokens` (§4.2 rule 7). The same tokens in both hosts -holds whenever the two hosts receive the same bodies and the same operations. +holds whenever the two hosts receive the same bodies, the same operations and send the same token counts. ## 6. The library diff --git a/internal/engine/body.go b/internal/engine/body.go index 19ea80e..08967d5 100644 --- a/internal/engine/body.go +++ b/internal/engine/body.go @@ -113,7 +113,7 @@ const mediaBytesPerToken = 250 func (b Body) cumMedia(i int) int { n := 0 for j := 0; j <= i && j < len(b.Pos); j++ { - n += int(b.Pos[j].Media) + n += int(min(b.Pos[j].Media, b.Pos[j].Bytes)) // a host-built Media above Bytes is capped } return n } @@ -123,6 +123,7 @@ func (b Body) cumMedia(i int) int { type ratio struct{ tokens, bytes int } // newRatio builds the ratio of a body that the provider billed for billed prompt tokens. +// The tokens clamp to 0 when the bill is below the media's fixed price; checks() reports that. func newRatio(billed int, b Body) ratio { last := len(b.Pos) - 1 media := b.cumMedia(last) diff --git a/internal/engine/compute.go b/internal/engine/compute.go index 7c3ec4e..dad2da6 100644 --- a/internal/engine/compute.go +++ b/internal/engine/compute.go @@ -248,6 +248,7 @@ type calcState struct { usage Usage noUsage bool ratio ratio + hostGap int // saved tokens that come from Size.Tokens beyond the ratio's figure (check 6) client, fwd Body clientBPs, fwdBPs []bp @@ -897,6 +898,11 @@ func (c *calcState) price(res *Result) { estimated = true } d := bt - at + if !it.op.DryRun { + // what the host's tokens add to the saving beyond the ratio's figure: the + // conservation check compares at the ratio (check 6) + c.hostGap += d - (c.ratio.tokensOf(sp.before.Bytes, sp.before.MediaBytes) - c.ratio.tokensOf(sp.after.Bytes, sp.after.MediaBytes)) + } if d > 0 { addTier(&p.Saved, mustFlat(c.client, sp.from), c.bstar, c.clientBPs, d) } else if d < 0 { @@ -943,6 +949,7 @@ func (c *calcState) conservation(res *Result) { } billed := c.usage.Input + c.usage.CacheRead + c.usage.CacheWrite want := c.ratio.upTo(c.client, len(c.client.Pos)-1) - billed + net -= c.hostGap // the ratio's figure: host tokens are exact and the bodies are not if d := want - net; d > tol+1 || d < -(tol+1) { res.Checks = append(res.Checks, CheckFailure{Check: "conservation", Detail: fmt.Sprintf("bill difference=%d ops=%d tokens", want, net)}) @@ -955,6 +962,10 @@ func tierSum(t TierTokens) int { return t.Input + t.CacheWrite5m + t.CacheWrite1 // are not run here (conservation is, in calc): attribution holds by construction, because the library // only gives a break the ChangeID of one of the request's operations. func (c *calcState) checks(res *Result) { + if media := c.fwd.cumMedia(len(c.fwd.Pos)-1) / mediaBytesPerToken; !c.noUsage && media > c.usage.Input+c.usage.CacheRead+c.usage.CacheWrite { + res.Checks = append(res.Checks, CheckFailure{Check: "media_tokens", + Detail: fmt.Sprintf("media=%d tokens billed=%d: the text ratio is 0", media, c.usage.Input+c.usage.CacheRead+c.usage.CacheWrite)}) + } var got [3]int seen := map[string]bool{} for _, it := range c.items { diff --git a/internal/engine/fingerprint.go b/internal/engine/fingerprint.go index 548a1c2..583e62f 100644 --- a/internal/engine/fingerprint.go +++ b/internal/engine/fingerprint.go @@ -2,6 +2,8 @@ package engine import ( "bytes" + "encoding/json" + "strings" "time" ) @@ -61,36 +63,107 @@ func NewBody(tools, system, messages [][]byte) Body { return b } -// mediaBytes counts the base64 payload of the media blocks in one position: the value of each -// "data" string that follows a "base64" source type, as in {"type":"base64","media_type":"image/png", -// "data":"..."}. It does not parse JSON, and the payload is never kept (R2). +// mediaBytes counts the base64 media payload in one position (R2: it keeps only the count). Two +// forms count: +// - an image or document block whose "source" has "type":"base64": the length of the source's +// "data" string (Anthropic form); +// - a "url" or "image_url" string that starts with "data:" and holds ";base64,": the bytes after +// the comma (OpenAI data-URL form). +// +// Text that only quotes "base64" or "data" is not media. A position that is not valid JSON counts +// what was read before the error. func mediaBytes(p []byte) int { - const marker = `"base64"` + dec := json.NewDecoder(bytes.NewReader(p)) n := 0 - for rest := p; ; { - i := bytes.Index(rest, []byte(marker)) - if i < 0 { + for { + if _, ok := walkValue(dec, &n); !ok { return n } - rest = rest[i+len(marker):] - j := bytes.Index(rest, []byte(`"data"`)) - if j < 0 { - return n + } +} + +// valueInfo is what a parent needs to know about one parsed value. +type valueInfo struct { + str string // the value, when it is a string + isStr bool + base64 int // the "data" length, when the value is a base64 source object; else 0 +} + +const dataURLMarker = ";base64," + +// dataURLPayload returns the payload length of a base64 data URL, or 0. +func dataURLPayload(s string) int { + if !strings.HasPrefix(s, "data:") { + return 0 + } + i := strings.Index(s, dataURLMarker) + if i < 0 { + return 0 + } + return len(s) - i - len(dataURLMarker) +} + +// walkValue reads one JSON value and adds the media it holds to *n. false means the input ended +// or is not valid JSON. +func walkValue(dec *json.Decoder, n *int) (valueInfo, bool) { + tok, err := dec.Token() + if err != nil { + return valueInfo{}, false + } + switch v := tok.(type) { + case string: + return valueInfo{str: v, isStr: true}, true + case json.Delim: + if v == '[' { + for dec.More() { + if _, ok := walkValue(dec, n); !ok { + return valueInfo{}, false + } + } + _, err := dec.Token() + return valueInfo{}, err == nil } - k := j + len(`"data"`) - // skip spaces and the colon, then the opening quote - for k < len(rest) && (rest[k] == ' ' || rest[k] == ':') { - k++ + return walkObject(dec, n) + } + return valueInfo{}, true +} + +// walkObject reads the members of an object whose "{" is read. It counts the payload of an image +// or document block's base64 source, and of a data URL under "url" or "image_url". It returns the +// payload length when the object itself is a base64 source, so its parent block can count it. +func walkObject(dec *json.Decoder, n *int) (valueInfo, bool) { + var typ string + var data valueInfo + var srcLen int + for dec.More() { + kt, err := dec.Token() + if err != nil { + return valueInfo{}, false } - if k >= len(rest) || rest[k] != '"' { - continue + key, _ := kt.(string) + child, ok := walkValue(dec, n) + if !ok { + return valueInfo{}, false } - k++ - end := bytes.IndexByte(rest[k:], '"') - if end < 0 { - return n + len(rest) - k + switch key { + case "type": + typ = child.str + case "data": + data = child + case "source": + srcLen = child.base64 + case "url", "image_url": + *n += dataURLPayload(child.str) } - n += end - rest = rest[k+end:] } + if _, err := dec.Token(); err != nil { + return valueInfo{}, false + } + if typ == "image" || typ == "document" { + *n += srcLen + } + if typ == "base64" && data.isStr { + return valueInfo{base64: len(data.str)}, true + } + return valueInfo{}, true } diff --git a/internal/engine/size_tokens_test.go b/internal/engine/size_tokens_test.go index 59d8aa3..f02232d 100644 --- a/internal/engine/size_tokens_test.go +++ b/internal/engine/size_tokens_test.go @@ -7,29 +7,31 @@ import ( // Issue 11: one bytes-per-token ratio misprices images and dense JSON. -// wrapper is the JSON text around the payload of imageBlock: text, 4 bytes a token here. -var wrapper = len(imageBlock(0)) - // imageBlock is a message that holds a base64 image of n payload bytes. func imageBlock(n int) []byte { return []byte(`{"type":"image","source":{"type":"base64","media_type":"image/png","data":"` + strings.Repeat("A", n) + `"}}`) } -// oneOp runs one request with one operation and returns the priced operation. -func oneOp(t *testing.T, body Body, op Operation, usage Usage) (Result, PricedOp) { +// wrapper is the JSON text around the payload of imageBlock: text, 4 bytes a token in these tests. +var wrapper = len(imageBlock(0)) + +func text(id string, n int) []byte { return []byte(id + strings.Repeat(".", n-len(id))) } + +// priceOne runs one request. The client body has the operation's range at its "before" size, and +// the forwarded body has it at its "after" size, so the sum and conservation checks hold. +func priceOne(t *testing.T, client, fwd [][]byte, op Operation, usage Usage) (Result, PricedOp) { t.Helper() ts := NewTrackers() - r := ts.BeginRequest(RequestMeta{RequestID: "r", Session: "s", At: t0}, body, nil) + r := ts.BeginRequest(RequestMeta{RequestID: "r", Session: "s", At: t0}, NewBody(nil, nil, client), nil) r.Add(op) - r.EndRequest(Thread{ID: "main"}, body, nil, exact5m) + r.EndRequest(Thread{ID: "main"}, NewBody(nil, nil, fwd), nil, exact5m) res := r.Compute(usage) return res, findOp(t, res, "o1") } func TestNewBodyCountsBase64MediaApart(t *testing.T) { - img := imageBlock(400000) - body := NewBody(nil, nil, [][]byte{img, []byte("plain text")}) + body := NewBody(nil, nil, [][]byte{imageBlock(400000), []byte("plain text")}) if got := body.Pos[0].Media; got != 400000 { t.Fatalf("Media = %d, want 400000", got) } @@ -38,30 +40,58 @@ func TestNewBodyCountsBase64MediaApart(t *testing.T) { } } +func TestMediaFormats(t *testing.T) { + cases := []struct { + name string + in string + want int + }{ + {"compact source block", `{"type":"image","source":{"type":"base64","media_type":"image/png","data":"AAAA"}}`, 4}, + {"pretty printed", "{\n\t\"type\": \"document\",\n\t\"source\": {\n\t\t\"type\": \"base64\",\n\t\t\"data\":\n\t\t\t\"AAAAAA\"\n\t}\n}", 6}, + {"source keys in other order", `{"source":{"data":"AAA","type":"base64"},"type":"image"}`, 3}, + {"block in a list", `[{"type":"text","text":"x"},{"type":"image","source":{"type":"base64","data":"AA"}}]`, 2}, + {"openai data url", `{"type":"image_url","image_url":{"url":"data:image/png;base64,AAAAA"}}`, 5}, + {"plain url", `{"type":"image_url","image_url":{"url":"https://x/y.png"}}`, 0}, + {"tool result that quotes base64 and data", `{"encoding":"base64","data":"AAAAAAAAAA"}`, 0}, + {"schema enum", `{"enum":["base64","hex"],"properties":{"data":{"type":"string"}}}`, 0}, + {"text block that mentions both", `{"type":"text","text":"use \"base64\" then \"data\": \"AAAA\""}`, 0}, + {"url source", `{"type":"image","source":{"type":"url","data":"AAAA"}}`, 0}, + {"base64 source in a text block", `{"type":"text","source":{"type":"base64","data":"AAAA"}}`, 0}, + {"data outside an image block", `{"type":"tool_result","content":{"type":"base64","data":"AAAA"}}`, 0}, + } + for _, c := range cases { + if got := mediaBytes([]byte(c.in)); got != c.want { + t.Errorf("%s: mediaBytes = %d, want %d", c.name, got, c.want) + } + } +} + // A 400,000-byte image (1600 tokens) and 3 text messages of 4000 bytes (1000 tokens each). // A cut of 4000 text bytes saves 1000 tokens. Before the fix the library booked 54. func TestImageDoesNotSkewTheTextRatio(t *testing.T) { - text := func(id string) []byte { return []byte(id + strings.Repeat(".", 4000-len(id))) } - body := NewBody(nil, nil, [][]byte{imageBlock(400000), text("a"), text("b"), text("c")}) - op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(1), + img := imageBlock(400000) + client := [][]byte{img, text("a", 4000), text("b", 4000), text("c", 4000)} + fwd := [][]byte{img, text("a", 4000), text("b", 4000), nil} + op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(3), SizeBefore: Size{Bytes: 4000}} - _, p := oneOp(t, body, op, Usage{Input: 1600 + (3*4000+wrapper)/4}) + billed := 1600 + (2*4000+wrapper)/4 + res, p := priceOne(t, client, fwd, op, Usage{Input: billed}) if got := tierSum(p.Saved); got < 998 || got > 1000 { // the wrapper text rounds - t.Fatalf("saved = %d tokens, want 1000", got) + t.Fatalf("saved = %d tokens, want about 1000", got) } if !p.Estimated { t.Fatal("a bytes conversion must stay Estimated") } + noChecks(t, res) } // 10,000 bytes of dense JSON are 5000 tokens. The request ratio would book 3500. func TestHostTokensAreUsedAsTheyAre(t *testing.T) { - json := []byte("j" + strings.Repeat("{", 9999)) - prose := []byte("p" + strings.Repeat(".", 39999)) - body := NewBody(nil, nil, [][]byte{json, prose}) + client := [][]byte{text("j", 10000), text("p", 40000)} + fwd := [][]byte{nil, text("p", 40000)} op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(0), SizeBefore: Size{Bytes: 10000, Tokens: 5000}} - _, p := oneOp(t, body, op, Usage{Input: 15000}) + _, p := priceOne(t, client, fwd, op, Usage{Input: 14000}) if got := tierSum(p.Saved); got != 5000 { t.Fatalf("saved = %d, want the host's 5000", got) } @@ -70,27 +100,70 @@ func TestHostTokensAreUsedAsTheyAre(t *testing.T) { } } +// Check 6 must pass on a correct report that uses Size.Tokens. Request 1 has an unknown phase and +// skips the check, so the gap shows only on request 2. +func TestConservationHoldsWithHostTokensOverTwoRequests(t *testing.T) { + dense, prose, extra := text("j", 10000), text("p", 40000), text("m", 400) + ts := NewTrackers() + r1 := ts.BeginRequest(RequestMeta{RequestID: "r1", Session: "s", At: t0}, NewBody(nil, nil, [][]byte{dense, prose}), nil) + r1.EndRequest(Thread{ID: "main"}, NewBody(nil, nil, [][]byte{dense, prose}), nil, exact5m) + noChecks(t, r1.Compute(Usage{Input: 14000})) + + small := text("j", 100) + r2 := ts.BeginRequest(RequestMeta{RequestID: "r2", Session: "s", At: t0.Add(1e9)}, + NewBody(nil, nil, [][]byte{dense, prose, extra}), nil) + r2.Add(Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(0), + SizeBefore: Size{Bytes: 10000, Tokens: 5000}, SizeAfter: Size{Bytes: 100, Tokens: 50}}) + r2.EndRequest(Thread{ID: "main"}, NewBody(nil, nil, [][]byte{small, prose, extra}), nil, exact5m) + res := r2.Compute(Usage{Input: 9500}) + noChecks(t, res) + if p := findOp(t, res, "o1"); tierSum(p.Saved) != 4950 || p.Estimated { + t.Fatalf("saved = %d estimated = %v, want the host's 4950, exact", tierSum(p.Saved), p.Estimated) + } +} + func TestMediaBytesOnASizeUseTheFixedMediaPrice(t *testing.T) { img := imageBlock(400000) - body := NewBody(nil, nil, [][]byte{img, []byte("t" + strings.Repeat(".", 3999))}) + client := [][]byte{img, text("t", 4000)} + fwd := [][]byte{nil, text("t", 4000)} op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(0), SizeBefore: Size{Bytes: len(img), MediaBytes: 400000}} - _, p := oneOp(t, body, op, Usage{Input: 1600 + (4000+wrapper)/4}) + res, p := priceOne(t, client, fwd, op, Usage{Input: 1000}) want := 1600 + wrapper/4 // the media at the fixed price, the JSON wrapper at the text ratio - if got := tierSum(p.Saved); got < want-2 || got > want { + if got := tierSum(p.Saved); got < want-2 || got > want+2 { t.Fatalf("saved = %d, want about %d", got, want) } + _ = res } func TestSizeWithMoreMediaThanBytesIsRefused(t *testing.T) { - body := NewBody(nil, nil, [][]byte{[]byte("abc")}) + // The reason name is negative_size; it covers any malformed Size. Left as is. + body := [][]byte{[]byte("abc")} op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(0), SizeBefore: Size{Bytes: 10, MediaBytes: 11}} - ts := NewTrackers() - r := ts.BeginRequest(RequestMeta{RequestID: "r", Session: "s", At: t0}, body, nil) - r.Add(op) - r.EndRequest(Thread{ID: "main"}, body, nil, exact5m) - if !hasUnpriced(r.Compute(Usage{Input: 3}), ReasonNegativeSize) { + res, _ := priceOne(t, body, body, op, Usage{Input: 3}) + if !hasUnpriced(res, ReasonNegativeSize) { t.Fatal("want negative_size") } } + +func TestHostBuiltMediaAboveBytesIsCapped(t *testing.T) { + b := Body{Start: [3]int{0, 0, 0}, Pos: []PosHash{{Hash: 1, Bytes: 100, Media: 5000}}} + if got := newRatio(50, b); got.bytes < 0 { + t.Fatalf("ratio bytes = %d, must not be negative", got.bytes) + } +} + +func TestBilledBelowMediaPriceIsAFailedCheck(t *testing.T) { + img := imageBlock(400000) // 1600 media tokens + body := [][]byte{img, text("t", 40)} + op := Operation{ID: "o1", Kind: KindRewrite, Operator: "mask", ChangeID: "mk", At: msg(1), + SizeBefore: Size{Bytes: 40}, SizeAfter: Size{Bytes: 40}} + res, _ := priceOne(t, body, body, op, Usage{Input: 100}) + for _, c := range res.Checks { + if c.Check == "media_tokens" { + return + } + } + t.Fatalf("Checks = %+v, want media_tokens", res.Checks) +} From f9343b7232ba4f4935992f99eb8eed47747e0ba4 Mon Sep 17 00:00:00 2001 From: DAVID BOAZ Date: Wed, 4 Nov 2026 17:24:24 +0200 Subject: [PATCH 3/3] fix: Price media at the media rate in the minimum-cacheable test writeBP estimated a breakpoint's prefix with ratio.of(cumBytes), which counts base64 media bytes at the text ratio. A media prefix below EntryFacts.MinCacheable was then judged cacheable, and a saving that was input got booked as a cache write. Use ratio.upTo, as every other conversion site does. Also say that an all-media body passes only when the bill equals the media price. Assisted-By: Claude (Anthropic AI) Signed-off-by: DAVID BOAZ --- docs/design.md | 2 +- internal/engine/compute.go | 2 +- internal/engine/mincacheable_test.go | 19 +++++++++++++++++++ internal/engine/size_tokens_test.go | 2 +- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/design.md b/docs/design.md index 9bff14d..c3aecaa 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1228,7 +1228,7 @@ check, the operator, the detail, and the request, session and thread IDs). `Prev is from 0.05 to 2.0. Base64 media bytes are not in the denominator, and the media price (1 token per 250 bytes) is not in the numerator. If it is outside that range, the usage does not fit the body, and the check `ratio_range` fails. A ratio of 0 (a body with text bytes and no billed prompt token) and an infinite ratio (billed tokens and an empty body) are outside - the range too. A body of media only has no text bytes: it passes when the bill covers the media price. A bill below + the range too. A body of media only has no text bytes: it passes only when the bill equals the media price. A bill above it fails this check, because the extra tokens fit no content. A bill below the media price fails this check as well as check 11. Without this check the ratio, and so the conservation check (check 6), agree with themselves for any usage. 10. **Breakpoints:** a request has at most 4 forwarded breakpoints (the Anthropic limit), and each TTL (client and diff --git a/internal/engine/compute.go b/internal/engine/compute.go index b2d099e..1fe2bd6 100644 --- a/internal/engine/compute.go +++ b/internal/engine/compute.go @@ -941,7 +941,7 @@ func (c *calcState) writeBP(body Body, bps []bp, p int) (ttl string, anyBP, ok b continue } anyBP = true - if (best < 0 || x.at < best) && (c.r.Facts.MinCacheable <= 0 || c.ratio.of(body.cumBytes(x.at)) >= c.r.Facts.MinCacheable) { + if (best < 0 || x.at < best) && (c.r.Facts.MinCacheable <= 0 || c.ratio.upTo(body, x.at) >= c.r.Facts.MinCacheable) { best, ttl = x.at, x.ttl } } diff --git a/internal/engine/mincacheable_test.go b/internal/engine/mincacheable_test.go index 8dd117a..c1ddfbc 100644 --- a/internal/engine/mincacheable_test.go +++ b/internal/engine/mincacheable_test.go @@ -165,3 +165,22 @@ func TestARestoreInsideBStarWithNoCacheTokensIsInput(t *testing.T) { t.Fatalf("Caused = %+v, want no read", c) } } + +// Media in the prefix prices at the media rate, not the text ratio (§5.1, rule 7). The image is +// 250,000 bytes, 1,000 tokens; the client prefix up to the breakpoint at m0 is 1000 + wrapper + 1000 + 600 +// = 2,667 tokens, below 4096. At the text ratio the image alone would count 250,000 and pass. +func TestAMediaPrefixBelowMinCacheableIsInput(t *testing.T) { + sysB := content([]blk{b("sys", 1000)}) + img := imageBlock(250000) + client := NewBody(nil, sysB, [][]byte{img, content([]blk{b("m0", 600)})[0]}) + fwd := NewBody(nil, sysB, [][]byte{img, content([]blk{b("m0", 300)})[0]}) + cbps := bps5(sys(0), msg(1)) + res := req{at: t0, client: client, clientBPs: cbps, + ops: []Operation{rewrite("a", "op", "c:1", msg(1), 600, 300)}, + fwd: fwd, fwdBPs: cbps, facts: minFacts(4096), + usage: Usage{CacheRead: 1000, Input: wrapper + 1000 + 300}}.run(NewTrackers()) + noChecks(t, res) + if s := findOp(t, res, "a").Saved; s.Input != 300 || s.CacheWrite5m != 0 { + t.Fatalf("Saved = %+v, want Input=300", s) + } +} diff --git a/internal/engine/size_tokens_test.go b/internal/engine/size_tokens_test.go index aeb2e31..331ea7a 100644 --- a/internal/engine/size_tokens_test.go +++ b/internal/engine/size_tokens_test.go @@ -353,7 +353,7 @@ func TestMediaHeavyRequestIsPriced(t *testing.T) { } } -// A body whose positions are all media is not an empty body: it prices when the bill covers the +// A body whose positions are all media is not an empty body: it prices when the bill equals the // media. func TestAllMediaBodyIsNotRefused(t *testing.T) { b := Body{Pos: []PosHash{{Hash: 1, Bytes: 2500, Media: 2500}}}