Feature/shimmy comm layers

.github/workflows/build-and-test.yml

@@ -60,10 +60,14 @@ jobs:
           wolframscript -code '
             PacletDirectoryLoad["."];
             Needs["LambdaFeedback`EvaluationFunctionToolkit`"];
-            report = TestReport["Tests/ServeFile.wlt"];
-            Print["Succeeded: ", report["TestsSucceededCount"]];
-            Print["Failed: ", report["TestsFailedCount"]];
-            If[report["TestsFailedCount"] > 0, Exit[1]];
+            testFiles = FileNames["*.wlt", "Tests"];
+            reports = TestReport /@ testFiles;
+            MapThread[
+              Print[#1, " -> Succeeded: ", #2["TestsSucceededCount"], " Failed: ", #2["TestsFailedCount"]] &,
+              {testFiles, reports}
+            ];
+            totalFailed = Total[#["TestsFailedCount"] & /@ reports];
+            If[totalFailed > 0, Exit[1]];
           '
 
       - name: Upload artifact

Bootstrap.wl

@@ -0,0 +1,21 @@
+(* ::Package:: *)
+
+(* Fixed entry point every Wolfram evaluation-function image runs, via
+   FUNCTION_ARGS set in evaluation-function-base/wolfram/Dockerfile
+   (which also git-clones this repo to LF_TOOLKIT_PATH, so this file ships
+   for free alongside the rest of the toolkit -- no separate COPY needed).
+
+   Handles the one bit of setup that has to happen before the toolkit's own
+   package loading can take over: pointing PacletDirectoryLoad at this
+   checkout so Needs can resolve LambdaFeedback`EvaluationFunctionToolkit`.
+   Every evaluation-function repo then just needs to provide evaluate.m/
+   preview.m defining evaluate`EvaluationFunction and preview`PreviewFunction
+   -- no per-repo wiring code required. *)
+
+PacletDirectoryLoad[Environment["LF_TOOLKIT_PATH"]];
+Needs["LambdaFeedback`EvaluationFunctionToolkit`"];
+
+<< "evaluate.m";
+<< "preview.m";
+
+ServeEvaluationFunction[evaluate`EvaluationFunction, preview`PreviewFunction]

Kernel/EvaluationFunctionToolkit.wl

@@ -6,227 +6,19 @@ BeginPackage["LambdaFeedback`EvaluationFunctionToolkit`"]
 
 Serve
 ServeFile
+ServeHttp
+ServeEvaluationFunction
 
 Begin["`Private`"]
 
-createError[code_, msg_, id_] := Module[{},
-  <|
-    "jsonrpc" -> "2.0",
-    "error" -> <|
-      "code" -> code,
-      "message" -> msg
-    |>,
-    "id" -> id
-  |>
-];
+$packageDir = DirectoryName[$InputFileName];
 
-createResponse[result_, id_] := Module[{},
-  <|
-    "jsonrpc" -> "2.0",
-    "result" -> result,
-    "id" -> id
-  |>
-];
-
-createErrorResponse[code_, msg_, id_] := Module[{},
-  ExportString[createError[code, msg, id], "JSON", "Compact" -> True]
-];
-
-(* Function to handle JSON-RPC 2.0 request and response *)
-handleJSONRPCRequest[eval_, req_] := Module[{method, params, id, result},
-  (* Get the request id *)
-  id = req["id"];
-  If[!IntegerQ[id],
-    Return[createError[-32600, "Missing request id", Null]]
-  ];
-
-  (* Return error if version is not "2.0" *)
-  version = req["jsonrpc"];
-  If[version =!= "2.0",
-    Return[createError[-32600, "Missing jsonrpc version", id]]
-  ];
-
-  (* Return error if method is not "eval" *)
-  method = req["method"];
-  If[method =!= "eval",
-    Return[createError[-32601, "Method not found", id]]
-  ];
-
-  params = req["params"];
-
-  (* Return error if params has not length of 1 *)
-  If[Length[params] != 1,
-    Return[createError[-32602, "Invalid params", id]]
-  ];
-
-  (* Return error if data is not an association *)
-  data = params[[1]];
-  If[!AssociationQ[data],
-    Return[createError[-32602, "Invalid params", id]]
-  ];
-
-  (* Return error if answer is empty *)
-  answer = Lookup[data, "answer", Null];
-  If[answer === Null,
-    Return[createError[-32602, "Missing answer", id]]
-  ];
-
-  (* Return error if response is empty *)
-  response = Lookup[data, "response", Null];
-  If[response === Null,
-    Return[createError[-32602, "Missing response", id]]
-  ];
-
-  (* Fall back to empty association if params is empty *)
-  params = Lookup[data, "params", <||>];
-
-  (* Run evaluation *)
-  result = eval[answer, response, params];
-
-  createResponse[result, id]
-];
-
-handleRequest[eval_, data_] := Module[{request, response},
-  (* Try to parse message as JSON *)
-  request = ImportString[data, "RawJSON"];
-  If[request === $Failed,
-    Return[createError[-32700, "Invalid JSON", Null]]
-  ];
-
-  (* Try to handle message *)
-  response = handleJSONRPCRequest[eval, request];
-  If[response === $Failed,
-    Return[createError[-32001, "Function error", request["id"]]],
-    Return[response]
-  ];
-];
-
-(* Function to handle incoming messages *)
-createMessageHandler[eval_] := Module[{handle},
-  handleMessage[msg_] := Module[{message},
-    (* Convert input bytes to string *)
-    str = ByteArrayToString[msg["DataByteArray"]];
-
-    (* Handle request *)
-    response = handleRequest[eval, str];
-
-    (* Get the source socket *)
-    socket = msg["SourceSocket"];
-
-    (* Stringify the response *)
-    responseStr = ExportString[response, "JSON", "Compact" -> True];
-    If[responseStr === $Failed,
-      WriteString[socket, createErrorResponse[-32000, "Encoding error", Null] <> "\n"];
-      Return[]
-    ];
-
-    (* Reply with the stringified response *)
-    WriteString[socket, responseStr <> "\n"];
-  ];
-
-  handleMessage
-]
-
-Serve[eval_] := Module[{},
-  socketAddress = Environment["EVAL_RPC_TCP_ADDRESS"];
-  If[socketAddress === $Failed, socketAddress = "127.0.0.1:7321"];
-
-  socket = SocketOpen[socketAddress];
-
-  handler = createMessageHandler[eval];
-
-  listener = SocketListen[socket, handler, RecordSeparators -> {"\n"}];
-
-  (* Print["Listening on ", socketAddress]; *)
-
-  While[True, Pause[60]];
-
-  (* Print["Closing connection"]; *)
-
-  DeleteObject[listener];
-  Close[socket];
-];
-
-(* ---- File-based transport ---- *)
-
-buildErrorResult[msg_] := <| "message" -> ToString[msg] |>;
-
-(* Catches Wolfram Messages raised by user code so a crash still produces a
-   JSON response instead of leaving the response file unwritten. *)
-safeCall[fn_, args___] := Quiet@Check[fn[args], $Failed];
-
-processEvalRequest[evalFn_, requestData_] := Module[
-  {params, answer, response, evalParams, result, errorMsg},
-  params = requestData["params"];
-  answer = params["answer"];
-  response = params["response"];
-  evalParams = params["params"];
-
-  Print["Running eval"];
-  result = safeCall[evalFn, answer, response, evalParams];
-
-  If[result === $Failed,
-    Return[<| "command" -> "eval", "error" -> buildErrorResult["Evaluation function raised an error"] |>]
-  ];
-
-  errorMsg = Lookup[result, "error", Null];
-  If[errorMsg =!= Null,
-    Return[<| "command" -> "eval", "error" -> buildErrorResult[errorMsg] |>]
-  ];
-
-  <|
-    "command" -> "eval",
-    "result" -> <|
-      "is_correct" -> result["is_correct"],
-      "feedback" -> result["feedback"]
-    |>
-  |>
-];
-
-processPreviewRequest[previewFn_, requestData_] := Module[
-  {params, response, previewParams, result},
-  params = requestData["params"];
-  response = params["response"];
-  previewParams = Lookup[params, "params", <||>];
-
-  Print["Running preview"];
-  result = safeCall[previewFn, response, previewParams];
-
-  If[result === $Failed,
-    Return[<| "command" -> "preview", "error" -> buildErrorResult["Preview function raised an error"] |>]
-  ];
-
-  <| "command" -> "preview", "result" -> <| "preview" -> result |> |>
-];
-
-processFileRequest[evalFn_, previewFn_, requestData_] := Module[{command},
-  command = Lookup[requestData, "command", "unknown"];
-  Which[
-    command === "eval", processEvalRequest[evalFn, requestData],
-    command === "preview", processPreviewRequest[previewFn, requestData],
-    True, <| "command" -> command, "error" -> buildErrorResult["Unknown command: " <> ToString[command]] |>
-  ]
-];
-
-ServeFile[evalFn_, previewFn_, requestPath_String, responsePath_String] := Module[
-  {requestData, responseData},
-  requestData = Import[requestPath, "JSON"] //. List :> Association;
-
-  Print["Input"];
-  Print[requestData];
-
-  responseData = processFileRequest[evalFn, previewFn, requestData];
-
-  Print["Output"];
-  Print[responseData];
-
-  Export[responsePath, responseData, "JSON", "Compact" -> True];
-];
-
-ServeFile[evalFn_, previewFn_] := Module[{argv},
-  argv = Rest[$ScriptCommandLine];
-  ServeFile[evalFn, previewFn, argv[[1]], argv[[2]]]
-];
+Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "Execution.wl"}]];
+Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "FileTransport.wl"}]];
+Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "JsonRpc.wl"}]];
+Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "TcpTransport.wl"}]];
+Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "HttpTransport.wl"}]];
+Get[FileNameJoin[{$packageDir, "EvaluationFunctionToolkit", "Dispatch.wl"}]];
 
 End[] (* End `Private` *)
 

Kernel/EvaluationFunctionToolkit/Dispatch.wl

@@ -0,0 +1,81 @@
+(* ::Package:: *)
+
+(* ---- Transport dispatch ----
+   Decides, from the shimmy-supplied EVAL_IO / EVAL_RPC_TRANSPORT environment
+   variables, which underlying Serve-family function a consumer's
+   evaluation_function.wl should call. The decision itself (dispatchTransport)
+   is a pure function of two strings, kept separate from environment reading,
+   so it can be unit-tested directly -- mirrors the safeCall/runEval split in
+   Execution.wl and the parse/dispatch split in JsonRpc.wl. *)
+
+(* Rpc sub-transports shimmy may report via EVAL_RPC_TRANSPORT that this
+   toolkit does not implement yet. Listed explicitly so the error can say
+   "this is a real transport shimmy supports, just not wired up here yet"
+   rather than lumping it in with a genuinely unrecognized value.
+
+   "stdio" belongs here, not as a dispatch target: Wolfram Engine has no
+   supported way to get a readable stream handle onto a process's real
+   inherited stdin -- ReadLine/BinaryReadList on "stdin" or "/dev/stdin"
+   both fail (confirmed on macOS and in the Linux worker image, over both
+   wolframscript and a bare WolframKernel, in every invocation mode tried:
+   -file, -script, -code, -run). The kernel's own top-level loop clearly
+   *can* read real stdin internally, but never exposes it as a Streams[]
+   object user code can Read from. Revisit only alongside a LibraryLink C
+   shim (raw read(2)/write(2) on fd 0/1), not as a pure-WL fix. *)
+$unimplementedRpcTransports = {"stdio", "ipc", "ws"};
+
+(* dispatchTransport[evalIO, rpcTransport] -> the Serve-family function to
+   call, or Failure[...] describing why none could be selected. Never talks
+   to the environment or the outside world. *)
+dispatchTransport[evalIO_String, rpcTransport_String] := Which[
+  evalIO =!= "rpc",
+    (* Shimmy's file adapter sets EVAL_IO=FILE (uppercase); running the
+       script directly without shimmy leaves it unset. Both, and any other
+       unrecognized value, fall back to the file transport -- preserving
+       today's evaluation_function.wl behavior of treating "not rpc" as
+       file-like. *)
+    ServeFile,
+  rpcTransport === "tcp",
+    Serve,
+  rpcTransport === "http",
+    ServeHttp,
+  MemberQ[$unimplementedRpcTransports, rpcTransport],
+    Failure["UnimplementedRpcTransport", <|
+      "MessageTemplate" -> "EVAL_RPC_TRANSPORT=" <> rpcTransport <>
+        " is a recognized Shimmy transport, but toolkit-wolfram does not implement it yet."
+    |>],
+  True,
+    Failure["UnknownRpcTransport", <|
+      "MessageTemplate" -> "Unrecognized EVAL_RPC_TRANSPORT value: " <> rpcTransport
+    |>]
+];
+
+(* Reads Environment[...] (normalizing unset/$Failed to "") and resolves the
+   dispatch target, without invoking it. Kept separate from
+   ServeEvaluationFunction so tests can exercise env-var reading without
+   also running a transport. *)
+resolveDispatchTarget[] := Module[{evalIO, rpcTransport},
+  evalIO = Environment["EVAL_IO"];
+  If[evalIO === $Failed, evalIO = ""];
+
+  rpcTransport = Environment["EVAL_RPC_TRANSPORT"];
+  If[rpcTransport === $Failed, rpcTransport = ""];
+
+  dispatchTransport[evalIO, rpcTransport]
+];
+
+(* Picks and runs the right Serve-family function for the current process's
+   environment. A grading worker silently hanging or misbehaving because of
+   an unimplemented/unrecognized transport is worse than a clear nonzero
+   exit shimmy's supervisor can observe and log -- so an unresolvable
+   transport fails the whole process rather than falling back to anything. *)
+ServeEvaluationFunction[evalFn_, previewFn_] := Module[{target},
+  target = resolveDispatchTarget[];
+
+  If[FailureQ[target],
+    Print["FATAL: ", target["Message"]];
+    Exit[1]
+  ];
+
+  target[evalFn, previewFn]
+];

Kernel/EvaluationFunctionToolkit/Execution.wl

@@ -0,0 +1,38 @@
+(* ::Package:: *)
+
+(* ---- Shared, transport-agnostic execution core ----
+   These functions call the user's eval/preview functions safely and return a
+   normalized outcome association (`"ok" -> True/False`), independent of how
+   the calling transport eventually formats that outcome on the wire. *)
+
+(* Catches Wolfram Messages raised by user code so a crash still produces a
+   normalized failure outcome instead of propagating. *)
+safeCall[fn_, args___] := Quiet@Check[fn[args], $Failed];
+
+runEval[evalFn_, answer_, response_, params_] := Module[{result, errorMsg},
+  result = safeCall[evalFn, answer, response, params];
+
+  If[result === $Failed,
+    Return[<| "ok" -> False, "message" -> "Evaluation function raised an error" |>]
+  ];
+
+  errorMsg = Lookup[result, "error", Null];
+  If[errorMsg =!= Null,
+    Return[<| "ok" -> False, "message" -> ToString[errorMsg] |>]
+  ];
+
+  <| "ok" -> True, "data" -> <|
+    "is_correct" -> result["is_correct"],
+    "feedback" -> result["feedback"]
+  |> |>
+];
+
+runPreview[previewFn_, response_, params_] := Module[{result},
+  result = safeCall[previewFn, response, params];
+
+  If[result === $Failed,
+    Return[<| "ok" -> False, "message" -> "Preview function raised an error" |>]
+  ];
+
+  <| "ok" -> True, "data" -> <| "preview" -> result |> |>
+];