From 981a297797a329640e5930b98067436c673c815a Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 3 Aug 2026 11:10:10 +0100 Subject: [PATCH 1/6] Add test suite for JSON-RPC `Serve` transport with evaluation and preview handling --- .github/workflows/build-and-test.yml | 12 +- Kernel/EvaluationFunctionToolkit.wl | 275 ++++++++++++++------------- README.md | 13 +- Tests/Serve.wlt | 76 ++++++++ 4 files changed, 240 insertions(+), 136 deletions(-) create mode 100644 Tests/Serve.wlt diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 20e6166..72ed86c 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -60,10 +60,14 @@ jobs: wolframscript -code ' PacletDirectoryLoad["."]; Needs["LambdaFeedback`EvaluationFunctionToolkit`"]; - report = TestReport["Tests/ServeFile.wlt"]; - Print["Succeeded: ", report["TestsSucceededCount"]]; - Print["Failed: ", report["TestsFailedCount"]]; - If[report["TestsFailedCount"] > 0, Exit[1]]; + testFiles = FileNames["*.wlt", "Tests"]; + reports = TestReport /@ testFiles; + MapThread[ + Print[#1, " -> Succeeded: ", #2["TestsSucceededCount"], " Failed: ", #2["TestsFailedCount"]] &, + {testFiles, reports} + ]; + totalFailed = Total[#["TestsFailedCount"] & /@ reports]; + If[totalFailed > 0, Exit[1]]; ' - name: Upload artifact diff --git a/Kernel/EvaluationFunctionToolkit.wl b/Kernel/EvaluationFunctionToolkit.wl index 6a87073..7ab1fe4 100644 --- a/Kernel/EvaluationFunctionToolkit.wl +++ b/Kernel/EvaluationFunctionToolkit.wl @@ -9,31 +9,146 @@ ServeFile Begin["`Private`"] -createError[code_, msg_, id_] := Module[{}, - <| - "jsonrpc" -> "2.0", - "error" -> <| - "code" -> code, - "message" -> msg - |>, - "id" -> id - |> +(* ---- Shared, transport-agnostic execution core ---- + These functions call the user's eval/preview functions safely and return a + normalized outcome association (`"ok" -> True/False`), independent of how + the calling transport eventually formats that outcome on the wire. *) + +(* Catches Wolfram Messages raised by user code so a crash still produces a + normalized failure outcome instead of propagating. *) +safeCall[fn_, args___] := Quiet@Check[fn[args], $Failed]; + +runEval[evalFn_, answer_, response_, params_] := Module[{result, errorMsg}, + result = safeCall[evalFn, answer, response, params]; + + If[result === $Failed, + Return[<| "ok" -> False, "message" -> "Evaluation function raised an error" |>] + ]; + + errorMsg = Lookup[result, "error", Null]; + If[errorMsg =!= Null, + Return[<| "ok" -> False, "message" -> ToString[errorMsg] |>] + ]; + + <| "ok" -> True, "data" -> <| + "is_correct" -> result["is_correct"], + "feedback" -> result["feedback"] + |> |> +]; + +runPreview[previewFn_, response_, params_] := Module[{result}, + result = safeCall[previewFn, response, params]; + + If[result === $Failed, + Return[<| "ok" -> False, "message" -> "Preview function raised an error" |>] + ]; + + <| "ok" -> True, "data" -> <| "preview" -> result |> |> ]; -createResponse[result_, id_] := Module[{}, - <| - "jsonrpc" -> "2.0", - "result" -> result, - "id" -> id - |> +(* ---- File-based transport ---- *) + +buildFileResponse[command_, outcome_] := If[outcome["ok"], + <| "command" -> command, "result" -> outcome["data"] |>, + <| "command" -> command, "error" -> <| "message" -> outcome["message"] |> |> ]; -createErrorResponse[code_, msg_, id_] := Module[{}, - ExportString[createError[code, msg, id], "JSON", "Compact" -> True] +processFileRequest[evalFn_, previewFn_, requestData_] := Module[{command, params}, + command = Lookup[requestData, "command", "unknown"]; + params = Lookup[requestData, "params", <||>]; + Which[ + command === "eval", + buildFileResponse["eval", runEval[ + evalFn, params["answer"], params["response"], Lookup[params, "params", <||>] + ]], + command === "preview", + buildFileResponse["preview", runPreview[ + previewFn, params["response"], Lookup[params, "params", <||>] + ]], + True, + <| "command" -> command, "error" -> <| "message" -> "Unknown command: " <> ToString[command] |> |> + ] +]; + +ServeFile[evalFn_, previewFn_, requestPath_String, responsePath_String] := Module[ + {requestData, responseData}, + requestData = Import[requestPath, "JSON"] //. List :> Association; + + Print["Input"]; + Print[requestData]; + + responseData = processFileRequest[evalFn, previewFn, requestData]; + + Print["Output"]; + Print[responseData]; + + Export[responsePath, responseData, "JSON", "Compact" -> True]; +]; + +ServeFile[evalFn_, previewFn_] := Module[{argv}, + argv = Rest[$ScriptCommandLine]; + ServeFile[evalFn, previewFn, argv[[1]], argv[[2]]] +]; + +(* ---- RPC (JSON-RPC 2.0) transport core, shared across rpc transports ---- + Note: unlike the file transport, shimmy's rpc adapter takes whatever comes + back in the JSON-RPC response's "result" field and forwards it verbatim as + {"command": method, "result": } -- it does not inspect it for a + nested "error" key. So domain errors and crashes here must surface as real + JSON-RPC-level errors, never nested inside "result". *) + +createError[code_, msg_, id_] := <| + "jsonrpc" -> "2.0", + "error" -> <| + "code" -> code, + "message" -> msg + |>, + "id" -> id +|>; + +createResponse[result_, id_] := <| + "jsonrpc" -> "2.0", + "result" -> result, + "id" -> id +|>; + +createErrorResponse[code_, msg_, id_] := + ExportString[createError[code, msg, id], "JSON", "Compact" -> True]; + +handleEvalRPC[evalFn_, data_, id_] := Module[{answer, response, evalParams, outcome}, + answer = Lookup[data, "answer", Null]; + If[answer === Null, Return[createError[-32602, "Missing answer", id]]]; + + response = Lookup[data, "response", Null]; + If[response === Null, Return[createError[-32602, "Missing response", id]]]; + + evalParams = Lookup[data, "params", <||>]; + + outcome = runEval[evalFn, answer, response, evalParams]; + + If[outcome["ok"], + createResponse[outcome["data"], id], + createError[-32000, outcome["message"], id] + ] +]; + +handlePreviewRPC[previewFn_, data_, id_] := Module[{response, previewParams, outcome}, + response = Lookup[data, "response", Null]; + If[response === Null, Return[createError[-32602, "Missing response", id]]]; + + previewParams = Lookup[data, "params", <||>]; + + outcome = runPreview[previewFn, response, previewParams]; + + If[outcome["ok"], + createResponse[outcome["data"], id], + createError[-32000, outcome["message"], id] + ] ]; (* Function to handle JSON-RPC 2.0 request and response *) -handleJSONRPCRequest[eval_, req_] := Module[{method, params, id, result}, +handleJSONRPCRequest[evalFn_, previewFn_, req_] := Module[ + {method, params, id, version, data}, (* Get the request id *) id = req["id"]; If[!IntegerQ[id], @@ -46,9 +161,9 @@ handleJSONRPCRequest[eval_, req_] := Module[{method, params, id, result}, Return[createError[-32600, "Missing jsonrpc version", id]] ]; - (* Return error if method is not "eval" *) + (* Return error if method is not "eval" or "preview" *) method = req["method"]; - If[method =!= "eval", + If[method =!= "eval" && method =!= "preview", Return[createError[-32601, "Method not found", id]] ]; @@ -65,36 +180,21 @@ handleJSONRPCRequest[eval_, req_] := Module[{method, params, id, result}, Return[createError[-32602, "Invalid params", id]] ]; - (* Return error if answer is empty *) - answer = Lookup[data, "answer", Null]; - If[answer === Null, - Return[createError[-32602, "Missing answer", id]] - ]; - - (* Return error if response is empty *) - response = Lookup[data, "response", Null]; - If[response === Null, - Return[createError[-32602, "Missing response", id]] - ]; - - (* Fall back to empty association if params is empty *) - params = Lookup[data, "params", <||>]; - - (* Run evaluation *) - result = eval[answer, response, params]; - - createResponse[result, id] + If[method === "eval", + handleEvalRPC[evalFn, data, id], + handlePreviewRPC[previewFn, data, id] + ] ]; -handleRequest[eval_, data_] := Module[{request, response}, +handleRequest[evalFn_, previewFn_, data_] := Module[{request, response}, (* Try to parse message as JSON *) - request = ImportString[data, "RawJSON"]; + request = Quiet[ImportString[data, "RawJSON"]]; If[request === $Failed, Return[createError[-32700, "Invalid JSON", Null]] ]; (* Try to handle message *) - response = handleJSONRPCRequest[eval, request]; + response = handleJSONRPCRequest[evalFn, previewFn, request]; If[response === $Failed, Return[createError[-32001, "Function error", request["id"]]], Return[response] @@ -102,13 +202,13 @@ handleRequest[eval_, data_] := Module[{request, response}, ]; (* Function to handle incoming messages *) -createMessageHandler[eval_] := Module[{handle}, - handleMessage[msg_] := Module[{message}, +createMessageHandler[evalFn_, previewFn_] := Module[{}, + handleMessage[msg_] := Module[{str, response, socket, responseStr}, (* Convert input bytes to string *) str = ByteArrayToString[msg["DataByteArray"]]; (* Handle request *) - response = handleRequest[eval, str]; + response = handleRequest[evalFn, previewFn, str]; (* Get the source socket *) socket = msg["SourceSocket"]; @@ -127,13 +227,13 @@ createMessageHandler[eval_] := Module[{handle}, handleMessage ] -Serve[eval_] := Module[{}, +Serve[evalFn_, previewFn_] := Module[{socketAddress, socket, handler, listener}, socketAddress = Environment["EVAL_RPC_TCP_ADDRESS"]; If[socketAddress === $Failed, socketAddress = "127.0.0.1:7321"]; socket = SocketOpen[socketAddress]; - handler = createMessageHandler[eval]; + handler = createMessageHandler[evalFn, previewFn]; listener = SocketListen[socket, handler, RecordSeparators -> {"\n"}]; @@ -147,87 +247,6 @@ Serve[eval_] := Module[{}, Close[socket]; ]; -(* ---- File-based transport ---- *) - -buildErrorResult[msg_] := <| "message" -> ToString[msg] |>; - -(* Catches Wolfram Messages raised by user code so a crash still produces a - JSON response instead of leaving the response file unwritten. *) -safeCall[fn_, args___] := Quiet@Check[fn[args], $Failed]; - -processEvalRequest[evalFn_, requestData_] := Module[ - {params, answer, response, evalParams, result, errorMsg}, - params = requestData["params"]; - answer = params["answer"]; - response = params["response"]; - evalParams = params["params"]; - - Print["Running eval"]; - result = safeCall[evalFn, answer, response, evalParams]; - - If[result === $Failed, - Return[<| "command" -> "eval", "error" -> buildErrorResult["Evaluation function raised an error"] |>] - ]; - - errorMsg = Lookup[result, "error", Null]; - If[errorMsg =!= Null, - Return[<| "command" -> "eval", "error" -> buildErrorResult[errorMsg] |>] - ]; - - <| - "command" -> "eval", - "result" -> <| - "is_correct" -> result["is_correct"], - "feedback" -> result["feedback"] - |> - |> -]; - -processPreviewRequest[previewFn_, requestData_] := Module[ - {params, response, previewParams, result}, - params = requestData["params"]; - response = params["response"]; - previewParams = Lookup[params, "params", <||>]; - - Print["Running preview"]; - result = safeCall[previewFn, response, previewParams]; - - If[result === $Failed, - Return[<| "command" -> "preview", "error" -> buildErrorResult["Preview function raised an error"] |>] - ]; - - <| "command" -> "preview", "result" -> <| "preview" -> result |> |> -]; - -processFileRequest[evalFn_, previewFn_, requestData_] := Module[{command}, - command = Lookup[requestData, "command", "unknown"]; - Which[ - command === "eval", processEvalRequest[evalFn, requestData], - command === "preview", processPreviewRequest[previewFn, requestData], - True, <| "command" -> command, "error" -> buildErrorResult["Unknown command: " <> ToString[command]] |> - ] -]; - -ServeFile[evalFn_, previewFn_, requestPath_String, responsePath_String] := Module[ - {requestData, responseData}, - requestData = Import[requestPath, "JSON"] //. List :> Association; - - Print["Input"]; - Print[requestData]; - - responseData = processFileRequest[evalFn, previewFn, requestData]; - - Print["Output"]; - Print[responseData]; - - Export[responsePath, responseData, "JSON", "Compact" -> True]; -]; - -ServeFile[evalFn_, previewFn_] := Module[{argv}, - argv = Rest[$ScriptCommandLine]; - ServeFile[evalFn, previewFn, argv[[1]], argv[[2]]] -]; - End[] (* End `Private` *) EndPackage[] diff --git a/README.md b/README.md index bef93b4..cba7415 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,10 @@ The toolkit exposes one function per Shimmy comms transport. Currently: (`FUNCTION_INTERFACE="file"`): reads a request JSON file and writes a response JSON file, as invoked by `wolframscript -f evaluation_function.wl request.json response.json`. -- `Serve[EvaluationFunction]` — a socket/JSON-RPC transport for the `eval` - method only. +- `Serve[EvaluationFunction, PreviewFunction]` — the `tcp` RPC transport + (`EVAL_RPC_TRANSPORT="tcp"`): a persistent JSON-RPC 2.0 socket server + supporting the `eval` and `preview` methods. `healthcheck` is not yet + implemented. More Shimmy transports (stdio, ipc) are expected to be added over time, mirroring [`toolkit-python`](https://github.com/lambda-feedback/toolkit-python)'s @@ -43,8 +45,11 @@ ServeFile[EvaluationFunction, PreviewFunction] functions are free to embed their own inline error/unavailable state. If `EvaluationFunction` or `PreviewFunction` raises a Wolfram error/message -(not a `Throw`/`Abort`), `ServeFile` catches it and returns a normal -`{"command", "error"}` JSON response rather than crashing. +(not a `Throw`/`Abort`), both `ServeFile` and `Serve` catch it rather than +crashing. `ServeFile` returns a normal `{"command", "error"}` JSON response; +`Serve` returns a JSON-RPC 2.0 error object (`{"error": {"code", "message"}}`), +since a nested `"error"` key inside the JSON-RPC `"result"` would not be +recognized as an error by Shimmy on the RPC transports. ## Development diff --git a/Tests/Serve.wlt b/Tests/Serve.wlt new file mode 100644 index 0000000..f0cdae0 --- /dev/null +++ b/Tests/Serve.wlt @@ -0,0 +1,76 @@ +(* ::Package:: *) + +Needs["LambdaFeedback`EvaluationFunctionToolkit`"] + +evalOk[answer_, response_, params_] := <| + "is_correct" -> True, "feedback" -> "Correct!", "error" -> Null +|>; + +evalFail[answer_, response_, params_] := <| + "is_correct" -> False, "feedback" -> "", "error" -> "bad answer" +|>; + +evalCrash[answer_, response_, params_] := 1/0; + +previewOk[response_, params_] := <|"latex" -> "x^2", "sympy" -> "x**2"|>; + +handleRPC[evalFn_, previewFn_, requestAssoc_] := Module[{requestStr}, + requestStr = ExportString[requestAssoc, "JSON", "Compact" -> True]; + LambdaFeedback`EvaluationFunctionToolkit`Private`handleRequest[evalFn, previewFn, requestStr] +]; + +VerificationTest[ + handleRPC[ + evalOk, previewOk, + <|"jsonrpc" -> "2.0", "method" -> "eval", "id" -> 1, + "params" -> {<|"answer" -> "x", "response" -> "x", "params" -> <||>|>}|> + ], + <|"jsonrpc" -> "2.0", "result" -> <|"is_correct" -> True, "feedback" -> "Correct!"|>, "id" -> 1|>, + TestID -> "Serve-eval-success" +] + +VerificationTest[ + handleRPC[ + evalFail, previewOk, + <|"jsonrpc" -> "2.0", "method" -> "eval", "id" -> 2, + "params" -> {<|"answer" -> "x", "response" -> "y", "params" -> <||>|>}|> + ], + <|"jsonrpc" -> "2.0", "error" -> <|"code" -> -32000, "message" -> "bad answer"|>, "id" -> 2|>, + TestID -> "Serve-eval-domain-error-not-nested-in-result" +] + +VerificationTest[ + handleRPC[ + evalCrash, previewOk, + <|"jsonrpc" -> "2.0", "method" -> "eval", "id" -> 3, + "params" -> {<|"answer" -> "x", "response" -> "y", "params" -> <||>|>}|> + ], + <|"jsonrpc" -> "2.0", "error" -> <|"code" -> -32000, "message" -> "Evaluation function raised an error"|>, "id" -> 3|>, + TestID -> "Serve-eval-crash-is-caught" +] + +VerificationTest[ + handleRPC[ + evalOk, previewOk, + <|"jsonrpc" -> "2.0", "method" -> "preview", "id" -> 4, + "params" -> {<|"response" -> "x^2", "params" -> <||>|>}|> + ], + <|"jsonrpc" -> "2.0", "result" -> <|"preview" -> <|"latex" -> "x^2", "sympy" -> "x**2"|>|>, "id" -> 4|>, + TestID -> "Serve-preview-success" +] + +VerificationTest[ + handleRPC[ + evalOk, previewOk, + <|"jsonrpc" -> "2.0", "method" -> "frobnicate", "id" -> 5, + "params" -> {<||>}|> + ], + <|"jsonrpc" -> "2.0", "error" -> <|"code" -> -32601, "message" -> "Method not found"|>, "id" -> 5|>, + TestID -> "Serve-unknown-method" +] + +VerificationTest[ + LambdaFeedback`EvaluationFunctionToolkit`Private`handleRequest[evalOk, previewOk, "not json"], + <|"jsonrpc" -> "2.0", "error" -> <|"code" -> -32700, "message" -> "Invalid JSON"|>, "id" -> Null|>, + TestID -> "Serve-malformed-json" +] From 4a70f33612db512771b3b403f17eb22cf81c15ad Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 3 Aug 2026 11:35:48 +0100 Subject: [PATCH 2/6] Add evaluation and transport framework with JSON-RPC, file, and TCP support --- Kernel/EvaluationFunctionToolkit/Execution.wl | 38 ++++++ .../FileTransport.wl | 45 +++++++ Kernel/EvaluationFunctionToolkit/JsonRpc.wl | 112 ++++++++++++++++++ .../EvaluationFunctionToolkit/TcpTransport.wl | 49 ++++++++ 4 files changed, 244 insertions(+) create mode 100644 Kernel/EvaluationFunctionToolkit/Execution.wl create mode 100644 Kernel/EvaluationFunctionToolkit/FileTransport.wl create mode 100644 Kernel/EvaluationFunctionToolkit/JsonRpc.wl create mode 100644 Kernel/EvaluationFunctionToolkit/TcpTransport.wl diff --git a/Kernel/EvaluationFunctionToolkit/Execution.wl b/Kernel/EvaluationFunctionToolkit/Execution.wl new file mode 100644 index 0000000..d99e052 --- /dev/null +++ b/Kernel/EvaluationFunctionToolkit/Execution.wl @@ -0,0 +1,38 @@ +(* ::Package:: *) + +(* ---- Shared, transport-agnostic execution core ---- + These functions call the user's eval/preview functions safely and return a + normalized outcome association (`"ok" -> True/False`), independent of how + the calling transport eventually formats that outcome on the wire. *) + +(* Catches Wolfram Messages raised by user code so a crash still produces a + normalized failure outcome instead of propagating. *) +safeCall[fn_, args___] := Quiet@Check[fn[args], $Failed]; + +runEval[evalFn_, answer_, response_, params_] := Module[{result, errorMsg}, + result = safeCall[evalFn, answer, response, params]; + + If[result === $Failed, + Return[<| "ok" -> False, "message" -> "Evaluation function raised an error" |>] + ]; + + errorMsg = Lookup[result, "error", Null]; + If[errorMsg =!= Null, + Return[<| "ok" -> False, "message" -> ToString[errorMsg] |>] + ]; + + <| "ok" -> True, "data" -> <| + "is_correct" -> result["is_correct"], + "feedback" -> result["feedback"] + |> |> +]; + +runPreview[previewFn_, response_, params_] := Module[{result}, + result = safeCall[previewFn, response, params]; + + If[result === $Failed, + Return[<| "ok" -> False, "message" -> "Preview function raised an error" |>] + ]; + + <| "ok" -> True, "data" -> <| "preview" -> result |> |> +]; diff --git a/Kernel/EvaluationFunctionToolkit/FileTransport.wl b/Kernel/EvaluationFunctionToolkit/FileTransport.wl new file mode 100644 index 0000000..b6d1975 --- /dev/null +++ b/Kernel/EvaluationFunctionToolkit/FileTransport.wl @@ -0,0 +1,45 @@ +(* ::Package:: *) + +(* ---- File-based transport ---- *) + +buildFileResponse[command_, outcome_] := If[outcome["ok"], + <| "command" -> command, "result" -> outcome["data"] |>, + <| "command" -> command, "error" -> <| "message" -> outcome["message"] |> |> +]; + +processFileRequest[evalFn_, previewFn_, requestData_] := Module[{command, params}, + command = Lookup[requestData, "command", "unknown"]; + params = Lookup[requestData, "params", <||>]; + Which[ + command === "eval", + buildFileResponse["eval", runEval[ + evalFn, params["answer"], params["response"], Lookup[params, "params", <||>] + ]], + command === "preview", + buildFileResponse["preview", runPreview[ + previewFn, params["response"], Lookup[params, "params", <||>] + ]], + True, + <| "command" -> command, "error" -> <| "message" -> "Unknown command: " <> ToString[command] |> |> + ] +]; + +ServeFile[evalFn_, previewFn_, requestPath_String, responsePath_String] := Module[ + {requestData, responseData}, + requestData = Import[requestPath, "JSON"] //. List :> Association; + + Print["Input"]; + Print[requestData]; + + responseData = processFileRequest[evalFn, previewFn, requestData]; + + Print["Output"]; + Print[responseData]; + + Export[responsePath, responseData, "JSON", "Compact" -> True]; +]; + +ServeFile[evalFn_, previewFn_] := Module[{argv}, + argv = Rest[$ScriptCommandLine]; + ServeFile[evalFn, previewFn, argv[[1]], argv[[2]]] +]; diff --git a/Kernel/EvaluationFunctionToolkit/JsonRpc.wl b/Kernel/EvaluationFunctionToolkit/JsonRpc.wl new file mode 100644 index 0000000..3fa6c54 --- /dev/null +++ b/Kernel/EvaluationFunctionToolkit/JsonRpc.wl @@ -0,0 +1,112 @@ +(* ::Package:: *) + +(* ---- RPC (JSON-RPC 2.0) transport core, shared across rpc transports ---- + Note: unlike the file transport, shimmy's rpc adapter takes whatever comes + back in the JSON-RPC response's "result" field and forwards it verbatim as + {"command": method, "result": } -- it does not inspect it for a + nested "error" key. So domain errors and crashes here must surface as real + JSON-RPC-level errors, never nested inside "result". *) + +createError[code_, msg_, id_] := <| + "jsonrpc" -> "2.0", + "error" -> <| + "code" -> code, + "message" -> msg + |>, + "id" -> id +|>; + +createResponse[result_, id_] := <| + "jsonrpc" -> "2.0", + "result" -> result, + "id" -> id +|>; + +createErrorResponse[code_, msg_, id_] := + ExportString[createError[code, msg, id], "JSON", "Compact" -> True]; + +handleEvalRPC[evalFn_, data_, id_] := Module[{answer, response, evalParams, outcome}, + answer = Lookup[data, "answer", Null]; + If[answer === Null, Return[createError[-32602, "Missing answer", id]]]; + + response = Lookup[data, "response", Null]; + If[response === Null, Return[createError[-32602, "Missing response", id]]]; + + evalParams = Lookup[data, "params", <||>]; + + outcome = runEval[evalFn, answer, response, evalParams]; + + If[outcome["ok"], + createResponse[outcome["data"], id], + createError[-32000, outcome["message"], id] + ] +]; + +handlePreviewRPC[previewFn_, data_, id_] := Module[{response, previewParams, outcome}, + response = Lookup[data, "response", Null]; + If[response === Null, Return[createError[-32602, "Missing response", id]]]; + + previewParams = Lookup[data, "params", <||>]; + + outcome = runPreview[previewFn, response, previewParams]; + + If[outcome["ok"], + createResponse[outcome["data"], id], + createError[-32000, outcome["message"], id] + ] +]; + +(* Function to handle JSON-RPC 2.0 request and response *) +handleJSONRPCRequest[evalFn_, previewFn_, req_] := Module[ + {method, params, id, version, data}, + (* Get the request id *) + id = req["id"]; + If[!IntegerQ[id], + Return[createError[-32600, "Missing request id", Null]] + ]; + + (* Return error if version is not "2.0" *) + version = req["jsonrpc"]; + If[version =!= "2.0", + Return[createError[-32600, "Missing jsonrpc version", id]] + ]; + + (* Return error if method is not "eval" or "preview" *) + method = req["method"]; + If[method =!= "eval" && method =!= "preview", + Return[createError[-32601, "Method not found", id]] + ]; + + params = req["params"]; + + (* Return error if params has not length of 1 *) + If[Length[params] != 1, + Return[createError[-32602, "Invalid params", id]] + ]; + + (* Return error if data is not an association *) + data = params[[1]]; + If[!AssociationQ[data], + Return[createError[-32602, "Invalid params", id]] + ]; + + If[method === "eval", + handleEvalRPC[evalFn, data, id], + handlePreviewRPC[previewFn, data, id] + ] +]; + +handleRequest[evalFn_, previewFn_, data_] := Module[{request, response}, + (* Try to parse message as JSON *) + request = Quiet[ImportString[data, "RawJSON"]]; + If[request === $Failed, + Return[createError[-32700, "Invalid JSON", Null]] + ]; + + (* Try to handle message *) + response = handleJSONRPCRequest[evalFn, previewFn, request]; + If[response === $Failed, + Return[createError[-32001, "Function error", request["id"]]], + Return[response] + ]; +]; diff --git a/Kernel/EvaluationFunctionToolkit/TcpTransport.wl b/Kernel/EvaluationFunctionToolkit/TcpTransport.wl new file mode 100644 index 0000000..8ac533b --- /dev/null +++ b/Kernel/EvaluationFunctionToolkit/TcpTransport.wl @@ -0,0 +1,49 @@ +(* ::Package:: *) + +(* ---- tcp RPC transport ---- *) + +(* Function to handle incoming messages *) +createMessageHandler[evalFn_, previewFn_] := Module[{}, + handleMessage[msg_] := Module[{str, response, socket, responseStr}, + (* Convert input bytes to string *) + str = ByteArrayToString[msg["DataByteArray"]]; + + (* Handle request *) + response = handleRequest[evalFn, previewFn, str]; + + (* Get the source socket *) + socket = msg["SourceSocket"]; + + (* Stringify the response *) + responseStr = ExportString[response, "JSON", "Compact" -> True]; + If[responseStr === $Failed, + WriteString[socket, createErrorResponse[-32000, "Encoding error", Null] <> "\n"]; + Return[] + ]; + + (* Reply with the stringified response *) + WriteString[socket, responseStr <> "\n"]; + ]; + + handleMessage +] + +Serve[evalFn_, previewFn_] := Module[{socketAddress, socket, handler, listener}, + socketAddress = Environment["EVAL_RPC_TCP_ADDRESS"]; + If[socketAddress === $Failed, socketAddress = "127.0.0.1:7321"]; + + socket = SocketOpen[socketAddress]; + + handler = createMessageHandler[evalFn, previewFn]; + + listener = SocketListen[socket, handler, RecordSeparators -> {"\n"}]; + + (* Print["Listening on ", socketAddress]; *) + + While[True, Pause[60]]; + + (* Print["Closing connection"]; *) + + DeleteObject[listener]; + Close[socket]; +]; From f80e2d4a8bd6999b419a50958ba77fa4eee50bdd Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 3 Aug 2026 11:35:53 +0100 Subject: [PATCH 3/6] Modularize evaluation and transport logic into separate files; update kernel loading. --- Kernel/EvaluationFunctionToolkit.wl | 241 +--------------------------- 1 file changed, 5 insertions(+), 236 deletions(-) diff --git a/Kernel/EvaluationFunctionToolkit.wl b/Kernel/EvaluationFunctionToolkit.wl index 7ab1fe4..eb01f56 100644 --- a/Kernel/EvaluationFunctionToolkit.wl +++ b/Kernel/EvaluationFunctionToolkit.wl @@ -9,243 +9,12 @@ ServeFile Begin["`Private`"] -(* ---- Shared, transport-agnostic execution core ---- - These functions call the user's eval/preview functions safely and return a - normalized outcome association (`"ok" -> True/False`), independent of how - the calling transport eventually formats that outcome on the wire. *) +$packageDir = DirectoryName[$InputFileName]; -(* Catches Wolfram Messages raised by user code so a crash still produces a - normalized failure outcome instead of propagating. *) -safeCall[fn_, args___] := Quiet@Check[fn[args], $Failed]; - -runEval[evalFn_, answer_, response_, params_] := Module[{result, errorMsg}, - result = safeCall[evalFn, answer, response, params]; - - If[result === $Failed, - Return[<| "ok" -> False, "message" -> "Evaluation function raised an error" |>] - ]; - - errorMsg = Lookup[result, "error", Null]; - If[errorMsg =!= Null, - Return[<| "ok" -> False, "message" -> ToString[errorMsg] |>] - ]; - - <| "ok" -> True, "data" -> <| - "is_correct" -> result["is_correct"], - "feedback" -> result["feedback"] - |> |> -]; - -runPreview[previewFn_, response_, params_] := Module[{result}, - result = safeCall[previewFn, response, params]; - - If[result === $Failed, - Return[<| "ok" -> False, "message" -> "Preview function raised an error" |>] - ]; - - <| "ok" -> True, "data" -> <| "preview" -> result |> |> -]; - -(* ---- File-based transport ---- *) - -buildFileResponse[command_, outcome_] := If[outcome["ok"], - <| "command" -> command, "result" -> outcome["data"] |>, - <| "command" -> command, "error" -> <| "message" -> outcome["message"] |> |> -]; - -processFileRequest[evalFn_, previewFn_, requestData_] := Module[{command, params}, - command = Lookup[requestData, "command", "unknown"]; - params = Lookup[requestData, "params", <||>]; - Which[ - command === "eval", - buildFileResponse["eval", runEval[ - evalFn, params["answer"], params["response"], Lookup[params, "params", <||>] - ]], - command === "preview", - buildFileResponse["preview", runPreview[ - previewFn, params["response"], Lookup[params, "params", <||>] - ]], - True, - <| "command" -> command, "error" -> <| "message" -> "Unknown command: " <> ToString[command] |> |> - ] -]; - -ServeFile[evalFn_, previewFn_, requestPath_String, responsePath_String] := Module[ - {requestData, responseData}, - requestData = Import[requestPath, "JSON"] //. List :> Association; - - Print["Input"]; - Print[requestData]; - - responseData = processFileRequest[evalFn, previewFn, requestData]; - - Print["Output"]; - Print[responseData]; - - Export[responsePath, responseData, "JSON", "Compact" -> True]; -]; - -ServeFile[evalFn_, previewFn_] := Module[{argv}, - argv = Rest[$ScriptCommandLine]; - ServeFile[evalFn, previewFn, argv[[1]], argv[[2]]] -]; - -(* ---- RPC (JSON-RPC 2.0) transport core, shared across rpc transports ---- - Note: unlike the file transport, shimmy's rpc adapter takes whatever comes - back in the JSON-RPC response's "result" field and forwards it verbatim as - {"command": method, "result": } -- it does not inspect it for a - nested "error" key. So domain errors and crashes here must surface as real - JSON-RPC-level errors, never nested inside "result". *) - -createError[code_, msg_, id_] := <| - "jsonrpc" -> "2.0", - "error" -> <| - "code" -> code, - "message" -> msg - |>, - "id" -> id -|>; - -createResponse[result_, id_] := <| - "jsonrpc" -> "2.0", - "result" -> result, - "id" -> id -|>; - -createErrorResponse[code_, msg_, id_] := - ExportString[createError[code, msg, id], "JSON", "Compact" -> True]; - -handleEvalRPC[evalFn_, data_, id_] := Module[{answer, response, evalParams, outcome}, - answer = Lookup[data, "answer", Null]; - If[answer === Null, Return[createError[-32602, "Missing answer", id]]]; - - response = Lookup[data, "response", Null]; - If[response === Null, Return[createError[-32602, "Missing response", id]]]; - - evalParams = Lookup[data, "params", <||>]; - - outcome = runEval[evalFn, answer, response, evalParams]; - - If[outcome["ok"], - createResponse[outcome["data"], id], - createError[-32000, outcome["message"], id] - ] -]; - -handlePreviewRPC[previewFn_, data_, id_] := Module[{response, previewParams, outcome}, - response = Lookup[data, "response", Null]; - If[response === Null, Return[createError[-32602, "Missing response", id]]]; - - previewParams = Lookup[data, "params", <||>]; - - outcome = runPreview[previewFn, response, previewParams]; - - If[outcome["ok"], - createResponse[outcome["data"], id], - createError[-32000, outcome["message"], id] - ] -]; - -(* Function to handle JSON-RPC 2.0 request and response *) -handleJSONRPCRequest[evalFn_, previewFn_, req_] := Module[ - {method, params, id, version, data}, - (* Get the request id *) - id = req["id"]; - If[!IntegerQ[id], - Return[createError[-32600, "Missing request id", Null]] - ]; - - (* Return error if version is not "2.0" *) - version = req["jsonrpc"]; - If[version =!= "2.0", - Return[createError[-32600, "Missing jsonrpc version", id]] - ]; - - (* Return error if method is not "eval" or "preview" *) - method = req["method"]; - If[method =!= "eval" && method =!= "preview", - Return[createError[-32601, "Method not found", id]] - ]; - - params = req["params"]; - - (* Return error if params has not length of 1 *) - If[Length[params] != 1, - Return[createError[-32602, "Invalid params", id]] - ]; - - (* Return error if data is not an association *) - data = params[[1]]; - If[!AssociationQ[data], - Return[createError[-32602, "Invalid params", id]] - ]; - - If[method === "eval", - handleEvalRPC[evalFn, data, id], - handlePreviewRPC[previewFn, data, id] - ] -]; - -handleRequest[evalFn_, previewFn_, data_] := Module[{request, response}, - (* Try to parse message as JSON *) - request = Quiet[ImportString[data, "RawJSON"]]; - If[request === $Failed, - Return[createError[-32700, "Invalid JSON", Null]] - ]; - - (* Try to handle message *) - response = handleJSONRPCRequest[evalFn, previewFn, request]; - If[response === $Failed, - Return[createError[-32001, "Function error", request["id"]]], - Return[response] - ]; -]; - -(* Function to handle incoming messages *) -createMessageHandler[evalFn_, previewFn_] := Module[{}, - handleMessage[msg_] := Module[{str, response, socket, responseStr}, - (* Convert input bytes to string *) - str = ByteArrayToString[msg["DataByteArray"]]; - - (* Handle request *) - response = handleRequest[evalFn, previewFn, str]; - - (* Get the source socket *) - socket = msg["SourceSocket"]; - - (* Stringify the response *) - responseStr = ExportString[response, "JSON", "Compact" -> True]; - If[responseStr === $Failed, - WriteString[socket, createErrorResponse[-32000, "Encoding error", Null] <> "\n"]; - Return[] - ]; - - (* Reply with the stringified response *) - WriteString[socket, responseStr <> "\n"]; - ]; - - handleMessage -] - -Serve[evalFn_, previewFn_] := Module[{socketAddress, socket, handler, listener}, - socketAddress = Environment["EVAL_RPC_TCP_ADDRESS"]; - If[socketAddress === $Failed, socketAddress = "127.0.0.1:7321"]; - - socket = SocketOpen[socketAddress]; - - handler = createMessageHandler[evalFn, previewFn]; - - listener = SocketListen[socket, handler, RecordSeparators -> {"\n"}]; - - (* Print["Listening on ", socketAddress]; *) - - While[True, Pause[60]]; - - (* Print["Closing connection"]; *) - - DeleteObject[listener]; - Close[socket]; -]; +Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "Execution.wl"}]]; +Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "FileTransport.wl"}]]; +Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "JsonRpc.wl"}]]; +Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "TcpTransport.wl"}]]; End[] (* End `Private` *) From f52626b4d3ef433421d68816523d35c1663323d0 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Mon, 3 Aug 2026 17:43:05 +0100 Subject: [PATCH 4/6] Add `ServeEvaluationFunction` as the toolkit's primary entry point and update README for usage and transport details. Bump version to 1.2.0. --- Kernel/EvaluationFunctionToolkit.wl | 2 + PacletInfo.wl | 2 +- README.md | 68 ++++++++++++++++++++++------- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/Kernel/EvaluationFunctionToolkit.wl b/Kernel/EvaluationFunctionToolkit.wl index eb01f56..12514a9 100644 --- a/Kernel/EvaluationFunctionToolkit.wl +++ b/Kernel/EvaluationFunctionToolkit.wl @@ -6,6 +6,7 @@ BeginPackage["LambdaFeedback`EvaluationFunctionToolkit`"] Serve ServeFile +ServeEvaluationFunction Begin["`Private`"] @@ -15,6 +16,7 @@ Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "Execution.wl"}]]; Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "FileTransport.wl"}]]; Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "JsonRpc.wl"}]]; Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "TcpTransport.wl"}]]; +Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "Dispatch.wl"}]]; End[] (* End `Private` *) diff --git a/PacletInfo.wl b/PacletInfo.wl index ba09660..c9772ae 100644 --- a/PacletInfo.wl +++ b/PacletInfo.wl @@ -7,7 +7,7 @@ PacletObject[ "Creator" -> "Andreas Pfurtscheller", "License" -> "MIT", "PublisherID" -> "LambdaFeedback", - "Version" -> "1.1.0", + "Version" -> "1.2.0", "WolframVersion" -> "13.+", "PrimaryContext" -> "LambdaFeedback`EvaluationFunctionToolkit`", "Extensions" -> { diff --git a/README.md b/README.md index cba7415..ecd0e36 100644 --- a/README.md +++ b/README.md @@ -6,20 +6,19 @@ A collection of utilities for creating Lambda Feedback evaluation functions for ## Usage -The toolkit exposes one function per Shimmy comms transport. Currently: - -- `ServeFile[EvaluationFunction, PreviewFunction]` — the file-based transport - (`FUNCTION_INTERFACE="file"`): reads a request JSON file and writes a - response JSON file, as invoked by `wolframscript -f evaluation_function.wl - request.json response.json`. -- `Serve[EvaluationFunction, PreviewFunction]` — the `tcp` RPC transport - (`EVAL_RPC_TRANSPORT="tcp"`): a persistent JSON-RPC 2.0 socket server - supporting the `eval` and `preview` methods. `healthcheck` is not yet - implemented. - -More Shimmy transports (stdio, ipc) are expected to be added over time, -mirroring [`toolkit-python`](https://github.com/lambda-feedback/toolkit-python)'s -`lf_toolkit/io/`. +Evaluation function repos built on +[`evaluation-function-base/wolfram`](https://github.com/lambda-feedback/evaluation-function-base) +don't need to call anything in this toolkit directly: that base image's +`FUNCTION_COMMAND`/`FUNCTION_ARGS` already point at this repo's +`Bootstrap.wl`, which loads the toolkit and wires it up automatically. Such a +repo only needs to provide an `evaluate.m` and `preview.m` (in the image's +working directory) defining `evaluate\`EvaluationFunction` and +`preview\`PreviewFunction` — see `Bootstrap.wl` for the exact contract. + +For anything else (custom wiring, local/manual testing, a different base +image), call `ServeEvaluationFunction` directly, which reads +Shimmy's environment-variable contract and dispatches to the right transport +— consumers don't need to know which transport Shimmy is running them under: ```wolfram Needs["LambdaFeedback`EvaluationFunctionToolkit`"] @@ -35,9 +34,44 @@ PreviewFunction[response_, params_] := <| "sympy" -> ToString[ToExpression[response], InputForm] |>; -ServeFile[EvaluationFunction, PreviewFunction] +ServeEvaluationFunction[EvaluationFunction, PreviewFunction] ``` +`ServeEvaluationFunction` reads: + +- `EVAL_IO` — `"rpc"` selects an RPC transport (below); anything else + (Shimmy's `"FILE"`, unset, or unrecognized) falls back to the file + transport. +- `EVAL_RPC_TRANSPORT` (only consulted when `EVAL_IO="rpc"`) — selects which + RPC transport to run. Currently only `"tcp"` is implemented. `"stdio"`, + `"ipc"`, `"http"`, `"ws"` are recognized Shimmy transports not yet + implemented in this toolkit; any other value is unrecognized. Either case + exits the process with a clear message and nonzero status rather than + silently falling back to a different transport — a grading worker doing + the wrong thing silently is worse than failing loudly where Shimmy's + supervisor can observe it. + +Internally, this dispatches to one function per Shimmy comms transport: + +- `ServeFile[EvaluationFunction, PreviewFunction]` — the file-based transport + (`FUNCTION_INTERFACE="file"`): reads a request JSON file and writes a + response JSON file, as invoked by `wolframscript -f evaluation_function.wl + request.json response.json`. +- `Serve[EvaluationFunction, PreviewFunction]` — the `tcp` RPC transport + (`EVAL_RPC_TRANSPORT="tcp"`): a persistent JSON-RPC 2.0 socket server + supporting the `eval` and `preview` methods. `healthcheck` is not yet + implemented. + +These remain exported for testing, but `ServeEvaluationFunction` is the +supported entry point for evaluation function repos — calling `ServeFile`/ +`Serve` directly means hand-rolling the transport-selection logic they exist +to avoid. + +More Shimmy transports (stdio, ipc) are expected to be added over time, +mirroring [`toolkit-python`](https://github.com/lambda-feedback/toolkit-python)'s +`lf_toolkit/io/`, each plugged into `ServeEvaluationFunction`'s dispatch as it +lands. + `EvaluationFunction` must return an association with `is_correct`, `feedback`, and `error` (`Null` on success, an error message otherwise). `PreviewFunction`'s return value is passed straight through under @@ -58,7 +92,9 @@ recognized as an error by Shimmy on the RPC transports. This toolkit is not published to the Wolfram Paclet Repository. Wolfram-based evaluation functions consume it by `git clone`ing a tagged version and pointing `PacletDirectoryLoad` at the checkout — see -`evaluation-function-base/wolfram/Dockerfile`'s `TOOLKIT_WOLFRAM_VERSION` build arg. +`evaluation-function-base/wolfram/Dockerfile`'s `TOOLKIT_WOLFRAM_VERSION` build +arg, and its `FUNCTION_COMMAND`/`FUNCTION_ARGS`, which run `Bootstrap.wl` from +that same checkout. To release a new version, tag the commit (`git tag vX.Y.Z && git push origin vX.Y.Z`) and bump `TOOLKIT_WOLFRAM_VERSION` in `evaluation-function-base/wolfram/Dockerfile`. From 5fac7795828d310e73577a9f3030983d716cf856 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Tue, 4 Aug 2026 11:25:41 +0100 Subject: [PATCH 5/6] Add Bootstrap.wl as a fixed entry point for evaluation functions; modularize dispatch logic into Dispatch.wl; implement transport selection fallback and testing framework for Shimmy integration. --- Bootstrap.wl | 21 +++++ Kernel/EvaluationFunctionToolkit/Dispatch.wl | 79 ++++++++++++++++++ README.md | 11 ++- Tests/ServeEvaluationFunction.wlt | 86 ++++++++++++++++++++ 4 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 Bootstrap.wl create mode 100644 Kernel/EvaluationFunctionToolkit/Dispatch.wl create mode 100644 Tests/ServeEvaluationFunction.wlt diff --git a/Bootstrap.wl b/Bootstrap.wl new file mode 100644 index 0000000..0cdb083 --- /dev/null +++ b/Bootstrap.wl @@ -0,0 +1,21 @@ +(* ::Package:: *) + +(* Fixed entry point every Wolfram evaluation-function image runs, via + FUNCTION_ARGS set in evaluation-function-base/wolfram/Dockerfile + (which also git-clones this repo to LF_TOOLKIT_PATH, so this file ships + for free alongside the rest of the toolkit -- no separate COPY needed). + + Handles the one bit of setup that has to happen before the toolkit's own + package loading can take over: pointing PacletDirectoryLoad at this + checkout so Needs can resolve LambdaFeedback`EvaluationFunctionToolkit`. + Every evaluation-function repo then just needs to provide evaluate.m/ + preview.m defining evaluate`EvaluationFunction and preview`PreviewFunction + -- no per-repo wiring code required. *) + +PacletDirectoryLoad[Environment["LF_TOOLKIT_PATH"]]; +Needs["LambdaFeedback`EvaluationFunctionToolkit`"]; + +<< "evaluate.m"; +<< "preview.m"; + +ServeEvaluationFunction[evaluate`EvaluationFunction, preview`PreviewFunction] diff --git a/Kernel/EvaluationFunctionToolkit/Dispatch.wl b/Kernel/EvaluationFunctionToolkit/Dispatch.wl new file mode 100644 index 0000000..70ed472 --- /dev/null +++ b/Kernel/EvaluationFunctionToolkit/Dispatch.wl @@ -0,0 +1,79 @@ +(* ::Package:: *) + +(* ---- Transport dispatch ---- + Decides, from the shimmy-supplied EVAL_IO / EVAL_RPC_TRANSPORT environment + variables, which underlying Serve-family function a consumer's + evaluation_function.wl should call. The decision itself (dispatchTransport) + is a pure function of two strings, kept separate from environment reading, + so it can be unit-tested directly -- mirrors the safeCall/runEval split in + Execution.wl and the parse/dispatch split in JsonRpc.wl. *) + +(* Rpc sub-transports shimmy may report via EVAL_RPC_TRANSPORT that this + toolkit does not implement yet. Listed explicitly so the error can say + "this is a real transport shimmy supports, just not wired up here yet" + rather than lumping it in with a genuinely unrecognized value. + + "stdio" belongs here, not as a dispatch target: Wolfram Engine has no + supported way to get a readable stream handle onto a process's real + inherited stdin -- ReadLine/BinaryReadList on "stdin" or "/dev/stdin" + both fail (confirmed on macOS and in the Linux worker image, over both + wolframscript and a bare WolframKernel, in every invocation mode tried: + -file, -script, -code, -run). The kernel's own top-level loop clearly + *can* read real stdin internally, but never exposes it as a Streams[] + object user code can Read from. Revisit only alongside a LibraryLink C + shim (raw read(2)/write(2) on fd 0/1), not as a pure-WL fix. *) +$unimplementedRpcTransports = {"stdio", "ipc", "http", "ws"}; + +(* dispatchTransport[evalIO, rpcTransport] -> the Serve-family function to + call, or Failure[...] describing why none could be selected. Never talks + to the environment or the outside world. *) +dispatchTransport[evalIO_String, rpcTransport_String] := Which[ + evalIO =!= "rpc", + (* Shimmy's file adapter sets EVAL_IO=FILE (uppercase); running the + script directly without shimmy leaves it unset. Both, and any other + unrecognized value, fall back to the file transport -- preserving + today's evaluation_function.wl behavior of treating "not rpc" as + file-like. *) + ServeFile, + rpcTransport === "tcp", + Serve, + MemberQ[$unimplementedRpcTransports, rpcTransport], + Failure["UnimplementedRpcTransport", <| + "Message" -> "EVAL_RPC_TRANSPORT=" <> rpcTransport <> + " is a recognized Shimmy transport, but toolkit-wolfram does not implement it yet." + |>], + True, + Failure["UnknownRpcTransport", <| + "Message" -> "Unrecognized EVAL_RPC_TRANSPORT value: " <> rpcTransport + |>] +]; + +(* Reads Environment[...] (normalizing unset/$Failed to "") and resolves the + dispatch target, without invoking it. Kept separate from + ServeEvaluationFunction so tests can exercise env-var reading without + also running a transport. *) +resolveDispatchTarget[] := Module[{evalIO, rpcTransport}, + evalIO = Environment["EVAL_IO"]; + If[evalIO === $Failed, evalIO = ""]; + + rpcTransport = Environment["EVAL_RPC_TRANSPORT"]; + If[rpcTransport === $Failed, rpcTransport = ""]; + + dispatchTransport[evalIO, rpcTransport] +]; + +(* Picks and runs the right Serve-family function for the current process's + environment. A grading worker silently hanging or misbehaving because of + an unimplemented/unrecognized transport is worse than a clear nonzero + exit shimmy's supervisor can observe and log -- so an unresolvable + transport fails the whole process rather than falling back to anything. *) +ServeEvaluationFunction[evalFn_, previewFn_] := Module[{target}, + target = resolveDispatchTarget[]; + + If[FailureQ[target], + Print["FATAL: ", target["Message"]]; + Exit[1] + ]; + + target[evalFn, previewFn] +]; diff --git a/README.md b/README.md index ecd0e36..180dd84 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,15 @@ supported entry point for evaluation function repos — calling `ServeFile`/ `Serve` directly means hand-rolling the transport-selection logic they exist to avoid. -More Shimmy transports (stdio, ipc) are expected to be added over time, -mirroring [`toolkit-python`](https://github.com/lambda-feedback/toolkit-python)'s +More Shimmy transports (stdio, ipc, http, ws) are expected to be added over +time, mirroring [`toolkit-python`](https://github.com/lambda-feedback/toolkit-python)'s `lf_toolkit/io/`, each plugged into `ServeEvaluationFunction`'s dispatch as it -lands. +lands. `stdio` in particular is blocked on Wolfram Engine, not just unwritten: +there is no supported way to get a readable stream handle onto a process's +real inherited stdin (`ReadLine`/`BinaryReadList` on `"stdin"` or +`"/dev/stdin"` both fail, confirmed on macOS and in the Linux worker image, +across every invocation mode tried). Revisiting it needs a LibraryLink C +shim, not a pure-WL fix. `EvaluationFunction` must return an association with `is_correct`, `feedback`, and `error` (`Null` on success, an error message otherwise). diff --git a/Tests/ServeEvaluationFunction.wlt b/Tests/ServeEvaluationFunction.wlt new file mode 100644 index 0000000..0bb1919 --- /dev/null +++ b/Tests/ServeEvaluationFunction.wlt @@ -0,0 +1,86 @@ +(* ::Package:: *) + +Needs["LambdaFeedback`EvaluationFunctionToolkit`"] + +dispatchTransport = LambdaFeedback`EvaluationFunctionToolkit`Private`dispatchTransport; +resolveDispatchTarget = LambdaFeedback`EvaluationFunctionToolkit`Private`resolveDispatchTarget; + +VerificationTest[ + dispatchTransport["FILE", ""], + ServeFile, + TestID -> "Dispatch-file-uppercase" +] + +VerificationTest[ + dispatchTransport["", ""], + ServeFile, + TestID -> "Dispatch-unset-falls-back-to-file" +] + +VerificationTest[ + dispatchTransport["garbage", ""], + ServeFile, + TestID -> "Dispatch-unrecognized-eval-io-falls-back-to-file" +] + +VerificationTest[ + dispatchTransport["rpc", "tcp"], + Serve, + TestID -> "Dispatch-rpc-tcp" +] + +VerificationTest[ + FailureQ[dispatchTransport["rpc", "stdio"]], + True, + TestID -> "Dispatch-rpc-stdio-not-yet-implemented" +] + +VerificationTest[ + FailureQ[dispatchTransport["rpc", "ipc"]], + True, + TestID -> "Dispatch-rpc-ipc-not-yet-implemented" +] + +VerificationTest[ + FailureQ[dispatchTransport["rpc", "http"]], + True, + TestID -> "Dispatch-rpc-http-not-yet-implemented" +] + +VerificationTest[ + FailureQ[dispatchTransport["rpc", "ws"]], + True, + TestID -> "Dispatch-rpc-ws-not-yet-implemented" +] + +VerificationTest[ + FailureQ[dispatchTransport["rpc", "bogus"]], + True, + TestID -> "Dispatch-rpc-unknown-transport" +] + +VerificationTest[ + FailureQ[dispatchTransport["rpc", ""]], + True, + TestID -> "Dispatch-rpc-missing-transport" +] + +(* Exercises Environment[] reading (resolveDispatchTarget), without invoking + Serve/ServeFile. Restores the vars it touches afterward since Tests/*.wlt + run in one shared kernel session per build-and-test.yml. *) +withEnv[vars_List, testFn_] := Module[{saved, result}, + saved = Environment /@ vars[[All, 1]]; + Scan[SetEnvironment[#[[1]] -> #[[2]]] &, vars]; + result = testFn[]; + MapThread[ + SetEnvironment[#1 -> If[#2 === $Failed, "", #2]] &, + {vars[[All, 1]], saved} + ]; + result +]; + +VerificationTest[ + withEnv[{"EVAL_IO" -> "rpc", "EVAL_RPC_TRANSPORT" -> "tcp"}, resolveDispatchTarget], + Serve, + TestID -> "Dispatch-resolveDispatchTarget-reads-environment" +] From 75b7f82f560a3445526cf2a024384094beb5cbab Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Tue, 4 Aug 2026 15:32:22 +0100 Subject: [PATCH 6/6] Add `ServeHttp` HTTP transport and tests for JSON-RPC 2.0 support; update dispatch logic and README. --- Kernel/EvaluationFunctionToolkit.wl | 2 + Kernel/EvaluationFunctionToolkit/Dispatch.wl | 8 +- .../HttpTransport.wl | 183 ++++++++++++++++ README.md | 42 ++-- Tests/ServeEvaluationFunction.wlt | 17 +- Tests/ServeHttp.wlt | 197 ++++++++++++++++++ 6 files changed, 428 insertions(+), 21 deletions(-) create mode 100644 Kernel/EvaluationFunctionToolkit/HttpTransport.wl create mode 100644 Tests/ServeHttp.wlt diff --git a/Kernel/EvaluationFunctionToolkit.wl b/Kernel/EvaluationFunctionToolkit.wl index 12514a9..a07c985 100644 --- a/Kernel/EvaluationFunctionToolkit.wl +++ b/Kernel/EvaluationFunctionToolkit.wl @@ -6,6 +6,7 @@ BeginPackage["LambdaFeedback`EvaluationFunctionToolkit`"] Serve ServeFile +ServeHttp ServeEvaluationFunction Begin["`Private`"] @@ -16,6 +17,7 @@ Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "Execution.wl"}]]; Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "FileTransport.wl"}]]; Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "JsonRpc.wl"}]]; Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "TcpTransport.wl"}]]; +Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "HttpTransport.wl"}]]; Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "Dispatch.wl"}]]; End[] (* End `Private` *) diff --git a/Kernel/EvaluationFunctionToolkit/Dispatch.wl b/Kernel/EvaluationFunctionToolkit/Dispatch.wl index 70ed472..2f1fa17 100644 --- a/Kernel/EvaluationFunctionToolkit/Dispatch.wl +++ b/Kernel/EvaluationFunctionToolkit/Dispatch.wl @@ -22,7 +22,7 @@ *can* read real stdin internally, but never exposes it as a Streams[] object user code can Read from. Revisit only alongside a LibraryLink C shim (raw read(2)/write(2) on fd 0/1), not as a pure-WL fix. *) -$unimplementedRpcTransports = {"stdio", "ipc", "http", "ws"}; +$unimplementedRpcTransports = {"stdio", "ipc", "ws"}; (* dispatchTransport[evalIO, rpcTransport] -> the Serve-family function to call, or Failure[...] describing why none could be selected. Never talks @@ -37,14 +37,16 @@ dispatchTransport[evalIO_String, rpcTransport_String] := Which[ ServeFile, rpcTransport === "tcp", Serve, + rpcTransport === "http", + ServeHttp, MemberQ[$unimplementedRpcTransports, rpcTransport], Failure["UnimplementedRpcTransport", <| - "Message" -> "EVAL_RPC_TRANSPORT=" <> rpcTransport <> + "MessageTemplate" -> "EVAL_RPC_TRANSPORT=" <> rpcTransport <> " is a recognized Shimmy transport, but toolkit-wolfram does not implement it yet." |>], True, Failure["UnknownRpcTransport", <| - "Message" -> "Unrecognized EVAL_RPC_TRANSPORT value: " <> rpcTransport + "MessageTemplate" -> "Unrecognized EVAL_RPC_TRANSPORT value: " <> rpcTransport |>] ]; diff --git a/Kernel/EvaluationFunctionToolkit/HttpTransport.wl b/Kernel/EvaluationFunctionToolkit/HttpTransport.wl new file mode 100644 index 0000000..32d49fb --- /dev/null +++ b/Kernel/EvaluationFunctionToolkit/HttpTransport.wl @@ -0,0 +1,183 @@ +(* ::Package:: *) + +(* ---- http RPC transport ---- + Shimmy's http transport (github.com/lambda-feedback/shimmy, + internal/execution/supervisor/adapter_rpc.go) is go-ethereum's generic + JSON-RPC HTTP client: it POSTs a single JSON-RPC 2.0 request per call to + the URL given via EVAL_RPC_HTTP_URL, and requires a 2xx status or it + surfaces an opaque transport-level error instead of reading the body as + JSON-RPC. So, exactly like the tcp transport, every outcome -- success, + domain error, caught crash -- must come back as HTTP 200 with a JSON-RPC + envelope; never as a non-2xx status. The client sends one request per + connection and is fine with the server closing the connection afterward + (it just redials next call), so this implementation makes no attempt at + keep-alive. + + Wolfram has no built-in local HTTP server primitive, so this hand-rolls + minimal HTTP/1.1 framing over a raw SocketListen, the same way + TcpTransport.wl hand-rolls JSON-RPC framing over a raw socket -- except + here there's no RecursionSeparators-style delimiter to lean on, since a + request's end is determined by the Content-Length header, not a fixed + terminator. accumulateHttpRequest is kept as a pure function of bytes in, + state out (no socket access) specifically so the framing logic is + testable without a live connection. *) + +(* accumulateHttpRequest[buffer, newBytes] -> the buffer's new state: + <|"complete" -> False, "buffer" -> ByteArray[...]|> if more bytes are + still needed (haven't seen the end of headers yet, or the body isn't + fully buffered yet), or <|"complete" -> True, "body" -> "..."|> once the + full request body has arrived. Never inspects method/path/other headers + -- Shimmy's real client only ever sends well-formed JSON-RPC POSTs, so + the only thing that matters here is where the body starts and ends. *) +accumulateHttpRequest[buffer_ByteArray, newBytes_ByteArray] := Module[ + {combined, bytes, sepMatch, sepStart, sepEnd, headerStr, contentLength, bodyEnd}, + combined = Join[buffer, newBytes]; + bytes = Normal[combined]; + + sepMatch = SequencePosition[bytes, {13, 10, 13, 10}, 1]; + If[sepMatch === {}, + Return[<|"complete" -> False, "buffer" -> combined|>] + ]; + + {sepStart, sepEnd} = First[sepMatch]; + headerStr = ByteArrayToString[ByteArray[bytes[[1 ;; sepStart - 1]]]]; + contentLength = parseContentLength[headerStr]; + + bodyEnd = sepEnd + contentLength; + If[Length[bytes] < bodyEnd, + Return[<|"complete" -> False, "buffer" -> combined|>] + ]; + + <|"complete" -> True, "body" -> ByteArrayToString[ByteArray[bytes[[sepEnd + 1 ;; bodyEnd]]]]|> +]; + +(* Case-insensitive Content-Length lookup out of a raw "\r\n"-joined header + block. Deliberately avoids ToExpression on header text (untrusted input) + -- pulls digits out with a regex and FromDigits instead. Missing/malformed + header defaults to 0, matching the "don't validate, let the JSON-RPC core + surface the resulting parse error" minimal-handling decision. *) +parseContentLength[headerStr_String] := Module[{lines, line, valueStr, digits}, + lines = StringSplit[headerStr, "\r\n"]; + line = SelectFirst[lines, StringMatchQ[#, RegularExpression["(?i)^content-length\\s*:.*"]] &, ""]; + If[line === "", Return[0]]; + + valueStr = StringTrim[StringReplace[line, RegularExpression["(?i)^content-length\\s*:"] -> ""]]; + digits = StringCases[valueStr, DigitCharacter ..]; + + If[digits === {}, 0, FromDigits[First[digits]]] +]; + +(* Extracts a "host:port" SocketOpen address out of EVAL_RPC_HTTP_URL. *) +parseHttpUrlHostPort[url_String] := Module[{parsed, host, port}, + parsed = URLParse[url]; + + host = Lookup[parsed, "Domain", "127.0.0.1"]; + If[host === None || host === "", host = "127.0.0.1"]; + + port = Lookup[parsed, "Port", 8000]; + If[port === None, port = 8000]; + + ToString[host] <> ":" <> ToString[port] +]; + +buildHttpResponse[bodyStr_String] := Module[{bodyBytes}, + bodyBytes = StringToByteArray[bodyStr, "UTF8"]; + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " <> + ToString[Length[bodyBytes]] <> "\r\nConnection: close\r\n\r\n" <> bodyStr +]; + +(* Holds one accumulation buffer per open socket, scoped to a single + createHttpMessageHandler call (i.e. one per ServeHttp listener), the same + closure-over-Module-local-state shape as TcpTransport.wl's + createMessageHandler. *) +createHttpMessageHandler[evalFn_, previewFn_] := Module[{buffers, handleMessage}, + buffers = <||>; + + handleMessage[msg_] := Module[{socket, incoming, state, jsonResponse, responseStr}, + socket = msg["SourceSocket"]; + incoming = msg["DataByteArray"]; + + (* Any event that doesn't carry data bytes (e.g. the connection closing) + just drops whatever partial state we had for this socket. *) + If[!ByteArrayQ[incoming], + buffers = KeyDrop[buffers, socket]; + Return[] + ]; + + state = accumulateHttpRequest[Lookup[buffers, socket, ByteArray[{}]], incoming]; + + If[!state["complete"], + buffers[socket] = state["buffer"]; + Return[] + ]; + + buffers = KeyDrop[buffers, socket]; + + jsonResponse = handleRequest[evalFn, previewFn, state["body"]]; + responseStr = ExportString[jsonResponse, "JSON", "Compact" -> True]; + If[responseStr === $Failed, + responseStr = createErrorResponse[-32000, "Encoding error", Null]; + ]; + + BinaryWrite[socket, StringToByteArray[buildHttpResponse[responseStr], "UTF8"]]; + Close[socket]; + ]; + + handleMessage +]; + +(* Opens the listening socket and wires up the handler, without blocking -- + split out from ServeHttp so tests can exercise a live listener without + invoking the infinite Pause loop below. + + Both SocketOpen and SocketListen return a Failure[...] (not $Failed) on + error -- e.g. SocketOpen on a port already in use -- so this checks head + types rather than === $Failed. A prior version of this code didn't check + either result at all: a failed bind would silently fall through to the + While loop below and sit there indefinitely looking alive to Shimmy (its + HTTP dial is lazy and never verifies connectivity at Start time) while + nothing was actually listening -- an undiagnosable hang from outside. + This also prints a positive "listening" confirmation on success, since + without it there's no way to tell from the process's own output whether + it ever got this far. *) +startHttpListener[evalFn_, previewFn_] := Module[{url, socketAddress, socket, handler, listener}, + url = Environment["EVAL_RPC_HTTP_URL"]; + If[url === $Failed || url === "", url = "http://127.0.0.1:8000/"]; + + socketAddress = parseHttpUrlHostPort[url]; + + socket = SocketOpen[socketAddress]; + If[Head[socket] =!= SocketObject, + Return[Failure["SocketOpenFailed", <| + "MessageTemplate" -> "Could not open a listening socket on " <> socketAddress <> + " (from EVAL_RPC_HTTP_URL=" <> url <> "): " <> ToString[socket] + |>]] + ]; + + handler = createHttpMessageHandler[evalFn, previewFn]; + listener = SocketListen[socket, handler]; + If[Head[listener] =!= SocketListener, + Close[socket]; + Return[Failure["SocketListenFailed", <| + "MessageTemplate" -> "Could not start listening on " <> socketAddress <> ": " <> ToString[listener] + |>]] + ]; + + Print["ServeHttp: listening on ", socketAddress, " (EVAL_RPC_HTTP_URL=", url, ")"]; + + <|"Socket" -> socket, "Listener" -> listener|> +]; + +ServeHttp[evalFn_, previewFn_] := Module[{running}, + running = startHttpListener[evalFn, previewFn]; + + If[FailureQ[running], + Print["FATAL: ", running["Message"]]; + Exit[1] + ]; + + While[True, Pause[60]]; + + DeleteObject[running["Listener"]]; + Close[running["Socket"]]; +]; diff --git a/README.md b/README.md index 180dd84..6c28d63 100644 --- a/README.md +++ b/README.md @@ -43,13 +43,13 @@ ServeEvaluationFunction[EvaluationFunction, PreviewFunction] (Shimmy's `"FILE"`, unset, or unrecognized) falls back to the file transport. - `EVAL_RPC_TRANSPORT` (only consulted when `EVAL_IO="rpc"`) — selects which - RPC transport to run. Currently only `"tcp"` is implemented. `"stdio"`, - `"ipc"`, `"http"`, `"ws"` are recognized Shimmy transports not yet - implemented in this toolkit; any other value is unrecognized. Either case - exits the process with a clear message and nonzero status rather than - silently falling back to a different transport — a grading worker doing - the wrong thing silently is worse than failing loudly where Shimmy's - supervisor can observe it. + RPC transport to run. `"tcp"` and `"http"` are implemented. `"stdio"`, + `"ipc"`, `"ws"` are recognized Shimmy transports not yet implemented in + this toolkit; any other value is unrecognized. Either case exits the + process with a clear message and nonzero status rather than silently + falling back to a different transport — a grading worker doing the wrong + thing silently is worse than failing loudly where Shimmy's supervisor can + observe it. Internally, this dispatches to one function per Shimmy comms transport: @@ -61,13 +61,24 @@ Internally, this dispatches to one function per Shimmy comms transport: (`EVAL_RPC_TRANSPORT="tcp"`): a persistent JSON-RPC 2.0 socket server supporting the `eval` and `preview` methods. `healthcheck` is not yet implemented. +- `ServeHttp[EvaluationFunction, PreviewFunction]` — the `http` RPC transport + (`EVAL_RPC_TRANSPORT="http"`): a JSON-RPC 2.0 server listening on the URL + given by `EVAL_RPC_HTTP_URL` (default `http://127.0.0.1:8000/` when unset, + for direct/manual invocation outside Shimmy). Each request is a single + `POST` whose body is a JSON-RPC 2.0 request; the response is written back + as the HTTP body. Shimmy's http transport is go-ethereum's generic + JSON-RPC HTTP client, which requires a 2xx status to read the body as + JSON-RPC at all — so, like `Serve`, every outcome (success, domain error, + caught crash) comes back as HTTP `200` with a JSON-RPC envelope, never a + non-2xx status. The connection is closed after each response; no + keep-alive. `healthcheck` is not yet implemented. These remain exported for testing, but `ServeEvaluationFunction` is the supported entry point for evaluation function repos — calling `ServeFile`/ -`Serve` directly means hand-rolling the transport-selection logic they exist -to avoid. +`Serve`/`ServeHttp` directly means hand-rolling the transport-selection logic +they exist to avoid. -More Shimmy transports (stdio, ipc, http, ws) are expected to be added over +More Shimmy transports (stdio, ipc, ws) are expected to be added over time, mirroring [`toolkit-python`](https://github.com/lambda-feedback/toolkit-python)'s `lf_toolkit/io/`, each plugged into `ServeEvaluationFunction`'s dispatch as it lands. `stdio` in particular is blocked on Wolfram Engine, not just unwritten: @@ -84,11 +95,12 @@ shim, not a pure-WL fix. functions are free to embed their own inline error/unavailable state. If `EvaluationFunction` or `PreviewFunction` raises a Wolfram error/message -(not a `Throw`/`Abort`), both `ServeFile` and `Serve` catch it rather than -crashing. `ServeFile` returns a normal `{"command", "error"}` JSON response; -`Serve` returns a JSON-RPC 2.0 error object (`{"error": {"code", "message"}}`), -since a nested `"error"` key inside the JSON-RPC `"result"` would not be -recognized as an error by Shimmy on the RPC transports. +(not a `Throw`/`Abort`), `ServeFile`, `Serve`, and `ServeHttp` all catch it +rather than crashing. `ServeFile` returns a normal `{"command", "error"}` +JSON response; `Serve` and `ServeHttp` return a JSON-RPC 2.0 error object +(`{"error": {"code", "message"}}`), since a nested `"error"` key inside the +JSON-RPC `"result"` would not be recognized as an error by Shimmy on the RPC +transports. ## Development diff --git a/Tests/ServeEvaluationFunction.wlt b/Tests/ServeEvaluationFunction.wlt index 0bb1919..d2745e0 100644 --- a/Tests/ServeEvaluationFunction.wlt +++ b/Tests/ServeEvaluationFunction.wlt @@ -35,6 +35,17 @@ VerificationTest[ TestID -> "Dispatch-rpc-stdio-not-yet-implemented" ] +(* Regression test: Failure[tag, <|"Message" -> "..."|>] does NOT make that + text retrievable via failure["Message"] -- Failure only recognizes + "MessageTemplate" for that. Silently building Failures with the wrong key + meant every "FATAL: ..." exit printed a useless generic + "A failure of type ... occurred." instead of the actual diagnostic. *) +VerificationTest[ + dispatchTransport["rpc", "stdio"]["Message"], + "EVAL_RPC_TRANSPORT=stdio is a recognized Shimmy transport, but toolkit-wolfram does not implement it yet.", + TestID -> "Dispatch-rpc-stdio-message-is-not-generic" +] + VerificationTest[ FailureQ[dispatchTransport["rpc", "ipc"]], True, @@ -42,9 +53,9 @@ VerificationTest[ ] VerificationTest[ - FailureQ[dispatchTransport["rpc", "http"]], - True, - TestID -> "Dispatch-rpc-http-not-yet-implemented" + dispatchTransport["rpc", "http"], + ServeHttp, + TestID -> "Dispatch-rpc-http" ] VerificationTest[ diff --git a/Tests/ServeHttp.wlt b/Tests/ServeHttp.wlt new file mode 100644 index 0000000..588cb10 --- /dev/null +++ b/Tests/ServeHttp.wlt @@ -0,0 +1,197 @@ +(* ::Package:: *) + +Needs["LambdaFeedback`EvaluationFunctionToolkit`"] + +accumulateHttpRequest = LambdaFeedback`EvaluationFunctionToolkit`Private`accumulateHttpRequest; +startHttpListener = LambdaFeedback`EvaluationFunctionToolkit`Private`startHttpListener; + +evalOk[answer_, response_, params_] := <| + "is_correct" -> True, "feedback" -> "Correct!", "error" -> Null +|>; + +previewOk[response_, params_] := <|"latex" -> "x^2", "sympy" -> "x**2"|>; + +handleRPC[evalFn_, previewFn_, requestAssoc_] := Module[{requestStr}, + requestStr = ExportString[requestAssoc, "JSON", "Compact" -> True]; + LambdaFeedback`EvaluationFunctionToolkit`Private`handleRequest[evalFn, previewFn, requestStr] +]; + +(* Builds a raw HTTP/1.1 POST request as a ByteArray for a given JSON body, + for feeding either directly into accumulateHttpRequest or over a real + socket in the end-to-end tests below. *) +buildRawHttpRequest[bodyStr_String] := Module[{bodyBytes, headerStr}, + bodyBytes = StringToByteArray[bodyStr, "UTF8"]; + headerStr = "POST / HTTP/1.1\r\nContent-Type: application/json\r\nContent-Length: " <> + ToString[Length[bodyBytes]] <> "\r\n\r\n"; + Join[StringToByteArray[headerStr, "UTF8"], bodyBytes] +]; + +(* ---- accumulateHttpRequest: pure framing-parser unit tests, no sockets ---- *) + +VerificationTest[ + Module[{req, state}, + req = buildRawHttpRequest["{\"jsonrpc\":\"2.0\"}"]; + state = accumulateHttpRequest[ByteArray[{}], req]; + {state["complete"], state["body"]} + ], + {True, "{\"jsonrpc\":\"2.0\"}"}, + TestID -> "ServeHttp-accumulate-single-chunk" +] + +VerificationTest[ + Module[{req, part1, part2, state1, state2}, + req = buildRawHttpRequest["{\"jsonrpc\":\"2.0\"}"]; + part1 = ByteArray[Normal[req][[1 ;; 10]]]; + part2 = ByteArray[Normal[req][[11 ;;]]]; + state1 = accumulateHttpRequest[ByteArray[{}], part1]; + state2 = accumulateHttpRequest[state1["buffer"], part2]; + {state1["complete"], state2["complete"], state2["body"]} + ], + {False, True, "{\"jsonrpc\":\"2.0\"}"}, + TestID -> "ServeHttp-accumulate-split-across-chunks" +] + +VerificationTest[ + Module[{req, headerAndPartial, rest, state1, state2}, + req = buildRawHttpRequest["{\"a\":1}"]; + (* split so the header/body boundary and the body itself both get cut + mid-content across chunks *) + headerAndPartial = ByteArray[Normal[req][[1 ;; -3]]]; + rest = ByteArray[Normal[req][[-2 ;;]]]; + state1 = accumulateHttpRequest[ByteArray[{}], headerAndPartial]; + state2 = accumulateHttpRequest[state1["buffer"], rest]; + {state1["complete"], state2["complete"], state2["body"]} + ], + {False, True, "{\"a\":1}"}, + TestID -> "ServeHttp-accumulate-body-split-mid-content" +] + +VerificationTest[ + Module[{req, state}, + req = StringToByteArray["GET / HTTP/1.1\r\nHost: x\r\n\r\n", "UTF8"]; + state = accumulateHttpRequest[ByteArray[{}], req]; + {state["complete"], state["body"]} + ], + {True, ""}, + TestID -> "ServeHttp-accumulate-missing-content-length-defaults-empty-body" +] + +(* ---- handleRequest reuse: confirms the shared JSON-RPC core is unchanged; + Tests/Serve.wlt already covers this logic thoroughly, so this is just + enough to confirm the http transport is wired to the same core. ---- *) + +VerificationTest[ + handleRPC[ + evalOk, previewOk, + <|"jsonrpc" -> "2.0", "method" -> "eval", "id" -> 1, + "params" -> {<|"answer" -> "x", "response" -> "x", "params" -> <||>|>}|> + ], + <|"jsonrpc" -> "2.0", "result" -> <|"is_correct" -> True, "feedback" -> "Correct!"|>, "id" -> 1|>, + TestID -> "ServeHttp-handleRequest-eval-success" +] + +VerificationTest[ + handleRPC[ + evalOk, previewOk, + <|"jsonrpc" -> "2.0", "method" -> "frobnicate", "id" -> 5, "params" -> {<||>}|> + ], + <|"jsonrpc" -> "2.0", "error" -> <|"code" -> -32601, "message" -> "Method not found"|>, "id" -> 5|>, + TestID -> "ServeHttp-handleRequest-unknown-method" +] + +(* startHttpListener returns a real Failure (not a silent hang) when the + socket can't be bound -- e.g. the port is already in use. Regression + coverage for both the failure detection itself, and for the + "MessageTemplate" vs "Message" Failure-construction bug: building a + Failure with a "Message" key does NOT make that text retrievable via + failure["Message"], so this also guards against that regressing. *) +VerificationTest[ + Module[{blocker, savedUrl, result}, + blocker = SocketOpen["127.0.0.1:8793"]; + savedUrl = Environment["EVAL_RPC_HTTP_URL"]; + SetEnvironment["EVAL_RPC_HTTP_URL" -> "http://127.0.0.1:8793/"]; + + result = startHttpListener[evalOk, previewOk]; + + Quiet[Close[blocker]]; + SetEnvironment["EVAL_RPC_HTTP_URL" -> If[savedUrl === $Failed, "", savedUrl]]; + + {FailureQ[result], StringContainsQ[result["Message"], "127.0.0.1:8793"]} + ], + {True, True}, + TestID -> "ServeHttp-startHttpListener-reports-bind-failure" +] + +(* ---- live end-to-end test: a real ServeHttp listener driven over a real + socket. This is the one piece none of the pure-function tests above can + cover -- the actual SocketListen wiring and HTTP response bytes written + back to a real client. ---- *) + +withHttpServer[port_Integer, testFn_] := Module[{savedUrl, running, result}, + savedUrl = Environment["EVAL_RPC_HTTP_URL"]; + SetEnvironment["EVAL_RPC_HTTP_URL" -> "http://127.0.0.1:" <> ToString[port] <> "/"]; + + running = startHttpListener[evalOk, previewOk]; + + result = testFn[]; + + Quiet[DeleteObject[running["Listener"]]]; + Quiet[Close[running["Socket"]]]; + SetEnvironment["EVAL_RPC_HTTP_URL" -> If[savedUrl === $Failed, "", savedUrl]]; + + result +]; + +readHttpResponse[socket_, timeoutSeconds_: 5] := Module[{deadline, buffer, chunk}, + deadline = AbsoluteTime[] + timeoutSeconds; + buffer = ByteArray[{}]; + While[AbsoluteTime[] < deadline, + If[SocketReadyQ[socket], + chunk = SocketReadMessage[socket]; + If[ByteArrayQ[chunk] && Length[chunk] > 0, + buffer = Join[buffer, chunk], + Break[] + ], + Pause[0.01] + ] + ]; + buffer +]; + +sendHttpRequest[port_Integer, bodyAssoc_Association] := withHttpServer[port, Function[ + Module[{requestStr, client, rawResponse, responseStr, headerEnd, statusLine, bodyStr}, + requestStr = ExportString[bodyAssoc, "JSON", "Compact" -> True]; + + client = SocketConnect["127.0.0.1:" <> ToString[port]]; + BinaryWrite[client, buildRawHttpRequest[requestStr]]; + + rawResponse = readHttpResponse[client]; + Quiet[Close[client]]; + + responseStr = ByteArrayToString[rawResponse, "UTF8"]; + statusLine = First[StringSplit[responseStr, "\r\n"]]; + headerEnd = First[First[StringPosition[responseStr, "\r\n\r\n"]]]; + bodyStr = StringDrop[responseStr, headerEnd + 3]; + + <|"status" -> statusLine, "body" -> ImportString[bodyStr, "RawJSON"]|> + ] +]]; + +VerificationTest[ + sendHttpRequest[8791, <|"jsonrpc" -> "2.0", "method" -> "eval", "id" -> 1, + "params" -> {<|"answer" -> "x", "response" -> "x", "params" -> <||>|>}|>], + <|"status" -> "HTTP/1.1 200 OK", + "body" -> <|"jsonrpc" -> "2.0", "result" -> <|"is_correct" -> True, "feedback" -> "Correct!"|>, "id" -> 1|>|>, + TestID -> "ServeHttp-e2e-eval-success" +] + +(* Confirms the critical Shimmy-compatibility contract end-to-end: a + JSON-RPC-level error still comes back as HTTP 200, never a non-2xx + status, since Shimmy's client treats non-2xx as an opaque transport + error rather than unpacking it as JSON-RPC. *) +VerificationTest[ + sendHttpRequest[8792, <|"jsonrpc" -> "2.0", "method" -> "frobnicate", "id" -> 5, "params" -> {<||>}|>], + <|"status" -> "HTTP/1.1 200 OK", + "body" -> <|"jsonrpc" -> "2.0", "error" -> <|"code" -> -32601, "message" -> "Method not found"|>, "id" -> 5|>|>, + TestID -> "ServeHttp-e2e-error-still-200" +]