Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion clients/python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "agent-eval-rpc"
version = "0.96.5"
version = "0.97.0"
description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client."
readme = "README.md"
requires-python = ">=3.10"
Expand Down
2 changes: 1 addition & 1 deletion clients/python/src/agent_eval_rpc/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
try:
__version__ = version("agent-eval-rpc")
except PackageNotFoundError:
__version__ = "0.96.5"
__version__ = "0.97.0"

__all__ = [
"Client",
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@tangle-network/agent-eval",
"version": "0.96.5",
"version": "0.97.0",
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
"homepage": "https://github.com/tangle-network/agent-eval#readme",
"repository": {
Expand Down
8 changes: 8 additions & 0 deletions src/campaign/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,14 @@ export interface ProposeContext<TFindings = unknown> {
* reflective / agentic generators both conform. They are proposers for the
* SAME loop, not separate loops. The loop body (`runOptimization`) and the
* gated promotion shell (`runImprovementLoop`) are proposer-agnostic.
*
* This is THE optimization proposer — every optimizer is a factory
* `xProposer(opts): SurfaceProposer` (`evolutionaryProposer`, `aceProposer`,
* `gepaProposer`, `skillOptProposer`, `traceAnalystProposer`, `haloProposer`,
* `memoryCurationProposer`, `fapoProposer`), all exported from `/campaign` and
* drivable by `selfImprove({ proposer })`. Not to be confused with the
* behavior-fuzzing `MutationProposer` (`fuzz/types`), a scenario generator for
* a different loop.
*/
export interface SurfaceProposer<TFindings = unknown> {
kind: string
Expand Down
13 changes: 10 additions & 3 deletions src/fuzz/fuzz-agent.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,14 @@ import { fuzzAgent } from './fuzz-agent'
import { composeGates } from './gates'
import { mutationProposer, noveltyObjective } from './policies'
import { makeExploreTools } from './tools'
import type { BehaviorSpace, Cell, Evaluation, Evaluator, ExploreOptions, Proposer } from './types'
import type {
BehaviorSpace,
Cell,
Evaluation,
Evaluator,
ExploreOptions,
MutationProposer,
} from './types'

interface Scn {
id: string
Expand Down Expand Up @@ -46,7 +53,7 @@ const seedsFor = (cell: Cell): Scn[] => [
},
]

const proposer: Proposer<Scn> = mutationProposer<Scn>({
const proposer: MutationProposer<Scn> = mutationProposer<Scn>({
scenarioId: (s) => s.id,
mutationsFor: () => [
{
Expand Down Expand Up @@ -163,7 +170,7 @@ describe('fuzzAgent (adversarial preset)', () => {
// allocation (base's mutation operator runs dry on easy cells, which is
// legitimate engine behavior, not an allocation skew).
let n = 0
const fresh: Proposer<Scn> = (ctx) =>
const fresh: MutationProposer<Scn> = (ctx) =>
Array.from({ length: ctx.count }, () => {
n++
return {
Expand Down
1 change: 1 addition & 0 deletions src/fuzz/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ export type {
ExploreEvent,
ExploreOptions,
Finding,
MutationProposer,
Objective,
ObjectiveContext,
ProposeContext,
Expand Down
8 changes: 4 additions & 4 deletions src/fuzz/policies.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
/**
* Shipped policies for the exploration engine.
*
* `Proposer` is a plain function type — an agent running a generator skill IS a
* proposer (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed. `mutationProposer`
* `MutationProposer` is a plain function type — an agent running a generator skill
* IS a proposer (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed. `mutationProposer`
* builds the deterministic, LLM-free one from mutation operators. Objectives are
* interfaces because the engine reads `kind` + `threshold` off them.
*/
Expand All @@ -11,10 +11,10 @@ import type { AdversarialMutation } from '../rl/adversarial'
import type {
Cell,
Evaluation,
MutationProposer,
Objective,
ObjectiveContext,
ProposeContext,
Proposer,
} from './types'

const clamp01 = (x: number): number => Math.max(0, Math.min(1, x))
Expand All @@ -29,7 +29,7 @@ const clamp01 = (x: number): number => Math.max(0, Math.min(1, x))
export function mutationProposer<S>(opts: {
mutationsFor: (cell: Cell) => AdversarialMutation<S>[]
scenarioId: (s: S) => string
}): Proposer<S> {
}): MutationProposer<S> {
return async (ctx: ProposeContext<S>): Promise<S[]> => {
const mutations = opts.mutationsFor(ctx.cell)
const parents = [...ctx.elites, ...ctx.seeds]
Expand Down
13 changes: 11 additions & 2 deletions src/fuzz/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,17 @@ export interface ProposeContext<S> {
* Produces candidate scenarios for a cell. A plain function — `mutationProposer`
* builds one from mutation operators; an agent running a generator skill IS one
* (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed.
*
* Distinct from the optimization `SurfaceProposer` (`campaign/types`): that one
* is the proposer in the surface-optimization loop (`runOptimization`); this one
* is the scenario generator in the behavior-fuzzing loop. `SurfaceProposer` is
* THE optimization proposer.
*/
export type Proposer<S> = (ctx: ProposeContext<S>) => Promise<S[]> | S[]
export type MutationProposer<S> = (ctx: ProposeContext<S>) => Promise<S[]> | S[]

/** @deprecated Renamed to `MutationProposer` to disambiguate from the
* optimization `SurfaceProposer`. Kept as an alias for back-compat. */
export type Proposer<S> = MutationProposer<S>

/**
* What "interesting" means. `interest` in [0,1]; a candidate is notable (gate-
Expand Down Expand Up @@ -249,7 +258,7 @@ export interface ExploreOptions<S> {
/** The input stratification plan. */
space: BehaviorSpace
/** Candidate generator. */
proposer: Proposer<S>
proposer: MutationProposer<S>
/** Runs the target → multi-objective `Evaluation`. */
evaluate: Evaluator<S>
/** Seed corpus per cell. */
Expand Down
Loading