Skip to content

Commit e45529e

Browse files
committed
refactor(appkit): unify eval CLI to a single --warehouse-id flag
The eval CLI carried two flags for one physical SQL warehouse: --warehouse-id (MLflow assessment writes to UC-backed traces) and --warehouse (managed-dataset reads). Collapse to --warehouse-id, feeding both paths; the MLflow write path keeps its MLFLOW_TRACING_SQL_WAREHOUSE_ID env fallback. Also apply oxfmt to eval files left unformatted by an earlier rebase. Signed-off-by: MarioCadenas <MarioCadenas@users.noreply.github.com>
1 parent 194466a commit e45529e

4 files changed

Lines changed: 20 additions & 10 deletions

File tree

‎packages/appkit/src/connectors/mlflow/auth.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
import { createWorkspaceClient, type WorkspaceClient } from "../../workspace-client";
1+
import {
2+
createWorkspaceClient,
3+
type WorkspaceClient,
4+
} from "../../workspace-client";
25

36
/** Resolved Databricks host + bearer token for the eval runner's REST calls. */
47
export interface DatabricksAuth {

‎packages/appkit/src/evals/run-evals.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { pathToFileURL } from "node:url";
2+
23
import { MlflowClient } from "../connectors/mlflow";
34
import type { WorkspaceClient } from "../workspace-client";
45
import { type DatasetRow, readEvalDataset } from "./dataset";
@@ -496,7 +497,15 @@ export async function runEvalsInDir(
496497
...options,
497498
timeoutMs: options.timeoutMs ?? configs.get(d.agent)?.timeoutMs,
498499
};
499-
await runDiscovered(d, index, total, runId, fileOptions, emit, fileResults);
500+
await runDiscovered(
501+
d,
502+
index,
503+
total,
504+
runId,
505+
fileOptions,
506+
emit,
507+
fileResults,
508+
);
500509
return fileResults;
501510
});
502511
const results = perFile.flat();

‎packages/appkit/src/evals/tests/run-eval.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { describe, expect, test, vi } from "vitest";
2+
23
import { defineEval } from "../define-eval";
34
import { includes } from "../matchers";
45
import { runEval } from "../run-eval";

‎packages/shared/src/cli/commands/agent/eval.ts‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import fs from "node:fs";
2+
23
import { Command, Option } from "commander";
34

45
interface EvalRunSummary {
@@ -109,7 +110,6 @@ interface EvalOptions {
109110
judgeModel?: string;
110111
concurrency?: number;
111112
warehouseId?: string;
112-
warehouse?: string;
113113
timeout?: string;
114114
retries?: string;
115115
minPassRate?: string;
@@ -243,8 +243,9 @@ async function runAgentEval(
243243
const auth: Auth = (await runner.resolveDatabricksAuth(credentials)) ?? {};
244244

245245
// Managed-dataset reads: a workspace client (same profile/host/token) + a SQL
246-
// warehouse. Only needed by evals that declare `dataset`.
247-
const warehouseId = opts.warehouse ?? process.env.DATABRICKS_WAREHOUSE_ID;
246+
// warehouse. Only needed by evals that declare `dataset`. Shares the single
247+
// --warehouse-id flag with the MLflow assessment-write path.
248+
const warehouseId = opts.warehouseId ?? process.env.DATABRICKS_WAREHOUSE_ID;
248249
const workspaceClient = runner.resolveWorkspaceClient(credentials);
249250

250251
// Runner-level default per-eval timeout (ms). A per-eval `timeoutMs` wins.
@@ -385,11 +386,7 @@ export const agentEvalCommand = new Command("eval")
385386
)
386387
.option(
387388
"--warehouse-id <id>",
388-
"SQL warehouse id for writing assessments to UC-backed experiments (default: MLFLOW_TRACING_SQL_WAREHOUSE_ID or DATABRICKS_WAREHOUSE_ID)",
389-
)
390-
.option(
391-
"--warehouse <id>",
392-
"SQL warehouse id for reading managed evaluation datasets (default: DATABRICKS_WAREHOUSE_ID)",
389+
"SQL warehouse id for reading managed eval datasets and writing assessments to UC-backed experiments (default: DATABRICKS_WAREHOUSE_ID, or MLFLOW_TRACING_SQL_WAREHOUSE_ID for assessments)",
393390
)
394391
.option(
395392
"--judge-model <endpoint>",

0 commit comments

Comments
 (0)