diff --git a/handler/chat.go b/handler/chat.go new file mode 100644 index 0000000..24e9e26 --- /dev/null +++ b/handler/chat.go @@ -0,0 +1,117 @@ +package handler + +import ( + "encoding/json" + "io" + "net/http" + + "github.com/lambda-feedback/shimmy/internal/server" + "github.com/lambda-feedback/shimmy/runtime" +) + +// ServeChat handles POST /chat. +func (h *MuEdHandler) ServeChat(w http.ResponseWriter, r *http.Request) { + if !h.checkAuth(w, r) { + return + } + + version, ok := h.checkMuEdVersion(w, r) + if !ok { + return + } + + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + body, err := io.ReadAll(r.Body) + if err != nil { + h.writeMuEdError(w, version, http.StatusBadRequest, "VALIDATION_ERROR", "Bad request", "failed to read body", nil) + return + } + + var chatReq runtime.MuEdChatRequest + if err := json.Unmarshal(body, &chatReq); err != nil { + h.writeMuEdError(w, version, http.StatusBadRequest, "VALIDATION_ERROR", "Bad request", "invalid request body", nil) + return + } + + reqData, err := runtime.MuEdBuildChatRequest(chatReq) + if err != nil { + h.writeMuEdError(w, version, http.StatusBadRequest, "VALIDATION_ERROR", "Bad request", err.Error(), nil) + return + } + + resp, err := h.runtime.Chat(r.Context(), runtime.ChatRequest{Data: reqData}) + if err != nil { + h.writeMuEdError(w, version, http.StatusInternalServerError, "INTERNAL_ERROR", "Internal server error", "chat failed", nil) + return + } + + resultMap, ok := resp.Data["result"].(map[string]any) + if !ok { + h.writeMuEdError(w, version, http.StatusInternalServerError, "INTERNAL_ERROR", "Internal server error", "invalid response from chat function", nil) + return + } + + chatResp, err := runtime.MuEdToChatResponse(resultMap) + if err != nil { + h.writeMuEdError(w, version, http.StatusInternalServerError, "INTERNAL_ERROR", "Internal server error", err.Error(), nil) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set(muEdVersionHeader, version) + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(chatResp) //nolint:errcheck +} + +// ServeChatHealth handles GET /chat/health. +func (h *MuEdHandler) ServeChatHealth(w http.ResponseWriter, r *http.Request) { + if !h.checkAuth(w, r) { + return + } + + version, ok := h.checkMuEdVersion(w, r) + if !ok { + return + } + + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + resp, err := h.runtime.ChatHealth(r.Context()) + if err != nil { + h.writeMuEdError(w, version, http.StatusInternalServerError, "INTERNAL_ERROR", "Internal server error", "chat health check failed", nil) + return + } + + resultMap, ok := resp.Data["result"].(map[string]any) + if !ok { + h.writeMuEdError(w, version, http.StatusInternalServerError, "INTERNAL_ERROR", "Internal server error", "invalid chat health response", nil) + return + } + + healthResp := runtime.MuEdToChatHealthResponse(resultMap) + + statusCode := http.StatusOK + if status, ok := healthResp["status"].(string); ok && status == string(runtime.MuEdChatHealthStatusUnavailable) { + statusCode = http.StatusServiceUnavailable + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set(muEdVersionHeader, version) + w.WriteHeader(statusCode) + json.NewEncoder(w).Encode(healthResp) //nolint:errcheck +} + +func NewMuEdChatRoute(handler *MuEdHandler) server.HttpHandlerResult { + return server.AsHttpHandler("/chat", http.HandlerFunc(handler.ServeChat)) +} + +func NewMuEdChatHealthRoute(handler *MuEdHandler) server.HttpHandlerResult { + return server.AsHttpHandler("/chat/health", http.HandlerFunc(handler.ServeChatHealth)) +} diff --git a/handler/chat_test.go b/handler/chat_test.go new file mode 100644 index 0000000..d6e717b --- /dev/null +++ b/handler/chat_test.go @@ -0,0 +1,330 @@ +package handler + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/lambda-feedback/shimmy/runtime" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// --- Helpers --- + +func chatRequestBody(t *testing.T) []byte { + t.Helper() + b, err := json.Marshal(map[string]any{ + "messages": []map[string]any{ + {"role": "USER", "content": "hello"}, + }, + }) + require.NoError(t, err) + return b +} + +func chatRuntimeResponse(role, content string) runtime.ChatResponse { + return runtime.ChatResponse{ + Data: map[string]any{ + "command": "chat", + "result": map[string]any{ + "output": map[string]any{ + "role": role, + "content": content, + }, + }, + }, + } +} + +// --- ServeChat tests --- + +func TestServeChat_Success(t *testing.T) { + mockRuntime := new(MockRuntime) + mockRuntime.On("Chat", mock.Anything, mock.Anything). + Return(chatRuntimeResponse("ASSISTANT", "Hello!"), nil) + + req := httptest.NewRequest(http.MethodPost, "/chat", bytes.NewReader(chatRequestBody(t))) + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChat(w, req) + + res := w.Result() + defer res.Body.Close() + body, _ := io.ReadAll(res.Body) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, "application/json", res.Header.Get("Content-Type")) + assert.Equal(t, "0.1.0", res.Header.Get("X-Api-Version")) + + var chatResp map[string]any + require.NoError(t, json.Unmarshal(body, &chatResp)) + output, ok := chatResp["output"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "ASSISTANT", output["role"]) + assert.Equal(t, "Hello!", output["content"]) + + mockRuntime.AssertExpectations(t) +} + +func TestServeChat_Unauthorized(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/chat", bytes.NewReader(chatRequestBody(t))) + req.Header.Set("api-key", "wrong") + w := httptest.NewRecorder() + + newMuEdHandler(nil, nil, "secret").ServeChat(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Result().StatusCode) +} + +func TestServeChat_MethodNotAllowed(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/chat", nil) + w := httptest.NewRecorder() + + newMuEdHandler(nil, nil, "").ServeChat(w, req) + + assert.Equal(t, http.StatusMethodNotAllowed, w.Result().StatusCode) +} + +func TestServeChat_InvalidJSON(t *testing.T) { + mockRuntime := new(MockRuntime) + + req := httptest.NewRequest(http.MethodPost, "/chat", bytes.NewReader([]byte("not json"))) + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChat(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Result().StatusCode) + mockRuntime.AssertNotCalled(t, "Chat", mock.Anything, mock.Anything) +} + +func TestServeChat_EmptyMessages(t *testing.T) { + mockRuntime := new(MockRuntime) + + body, _ := json.Marshal(map[string]any{"messages": []any{}}) + req := httptest.NewRequest(http.MethodPost, "/chat", bytes.NewReader(body)) + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChat(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Result().StatusCode) + mockRuntime.AssertNotCalled(t, "Chat", mock.Anything, mock.Anything) +} + +func TestServeChat_RuntimeError(t *testing.T) { + mockRuntime := new(MockRuntime) + mockRuntime.On("Chat", mock.Anything, mock.Anything). + Return(runtime.ChatResponse{}, errors.New("chat failed")) + + req := httptest.NewRequest(http.MethodPost, "/chat", bytes.NewReader(chatRequestBody(t))) + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChat(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Result().StatusCode) + mockRuntime.AssertExpectations(t) +} + +// --- ServeChatHealth tests --- + +func TestServeChatHealth_Success(t *testing.T) { + mockRuntime := new(MockRuntime) + mockRuntime.On("ChatHealth", mock.Anything).Return(runtime.ChatResponse{ + Data: map[string]any{ + "command": "chat/health", + "result": map[string]any{ + "status": "OK", + "capabilities": map[string]any{"supportsChat": true}, + "supportedLanguages": []any{}, + "supportedModels": []any{}, + "supportedAPIVersions": []any{}, + }, + }, + }, nil) + + req := httptest.NewRequest(http.MethodGet, "/chat/health", nil) + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChatHealth(w, req) + + res := w.Result() + defer res.Body.Close() + raw, _ := io.ReadAll(res.Body) + + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, "application/json", res.Header.Get("Content-Type")) + assert.Equal(t, "0.1.0", res.Header.Get("X-Api-Version")) + + var result map[string]any + require.NoError(t, json.Unmarshal(raw, &result)) + assert.Equal(t, "OK", result["status"]) + + mockRuntime.AssertExpectations(t) +} + +func TestServeChatHealth_Unauthorized(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/chat/health", nil) + req.Header.Set("api-key", "wrong") + w := httptest.NewRecorder() + + newMuEdHandler(nil, nil, "secret").ServeChatHealth(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Result().StatusCode) +} + +func TestServeChatHealth_MethodNotAllowed(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/chat/health", nil) + w := httptest.NewRecorder() + + newMuEdHandler(nil, nil, "").ServeChatHealth(w, req) + + assert.Equal(t, http.StatusMethodNotAllowed, w.Result().StatusCode) +} + +func TestServeChatHealth_Unavailable(t *testing.T) { + mockRuntime := new(MockRuntime) + mockRuntime.On("ChatHealth", mock.Anything).Return(runtime.ChatResponse{ + Data: map[string]any{ + "result": map[string]any{ + "status": "UNAVAILABLE", + }, + }, + }, nil) + + req := httptest.NewRequest(http.MethodGet, "/chat/health", nil) + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChatHealth(w, req) + + assert.Equal(t, http.StatusServiceUnavailable, w.Result().StatusCode) + mockRuntime.AssertExpectations(t) +} + +func TestServeChatHealth_RuntimeError(t *testing.T) { + mockRuntime := new(MockRuntime) + mockRuntime.On("ChatHealth", mock.Anything). + Return(runtime.ChatResponse{}, errors.New("worker unavailable")) + + req := httptest.NewRequest(http.MethodGet, "/chat/health", nil) + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChatHealth(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Result().StatusCode) + mockRuntime.AssertExpectations(t) +} + +// --- Version header tests (ServeChat) --- + +func TestServeChat_AbsentVersionHeader(t *testing.T) { + mockRuntime := new(MockRuntime) + mockRuntime.On("Chat", mock.Anything, mock.Anything). + Return(chatRuntimeResponse("ASSISTANT", "hi"), nil) + + req := httptest.NewRequest(http.MethodPost, "/chat", bytes.NewReader(chatRequestBody(t))) + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChat(w, req) + + res := w.Result() + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, "0.1.0", res.Header.Get("X-Api-Version")) +} + +func TestServeChat_SupportedVersionHeader(t *testing.T) { + mockRuntime := new(MockRuntime) + mockRuntime.On("Chat", mock.Anything, mock.Anything). + Return(chatRuntimeResponse("ASSISTANT", "hi"), nil) + + req := httptest.NewRequest(http.MethodPost, "/chat", bytes.NewReader(chatRequestBody(t))) + req.Header.Set("X-Api-Version", "0.1.0") + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChat(w, req) + + res := w.Result() + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, "0.1.0", res.Header.Get("X-Api-Version")) +} + +func TestServeChat_UnsupportedVersionHeader(t *testing.T) { + mockRuntime := new(MockRuntime) + + req := httptest.NewRequest(http.MethodPost, "/chat", bytes.NewReader(chatRequestBody(t))) + req.Header.Set("X-Api-Version", "99.0.0") + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChat(w, req) + + res := w.Result() + defer res.Body.Close() + raw, _ := io.ReadAll(res.Body) + + assert.Equal(t, http.StatusNotAcceptable, res.StatusCode) + assert.Equal(t, "0.1.0", res.Header.Get("X-Api-Version")) + + var body map[string]any + require.NoError(t, json.Unmarshal(raw, &body)) + assert.Equal(t, "VERSION_NOT_SUPPORTED", body["code"]) + details, ok := body["details"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "99.0.0", details["requestedVersion"]) + + mockRuntime.AssertNotCalled(t, "Chat", mock.Anything, mock.Anything) +} + +// --- Version header tests (ServeChatHealth) --- + +func TestServeChatHealth_AbsentVersionHeader(t *testing.T) { + mockRuntime := new(MockRuntime) + mockRuntime.On("ChatHealth", mock.Anything).Return(runtime.ChatResponse{ + Data: map[string]any{ + "command": "chat/health", + "result": map[string]any{ + "status": "OK", + "capabilities": map[string]any{"supportsChat": true}, + "supportedLanguages": []any{}, + "supportedModels": []any{}, + "supportedAPIVersions": []any{}, + }, + }, + }, nil) + + req := httptest.NewRequest(http.MethodGet, "/chat/health", nil) + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChatHealth(w, req) + + res := w.Result() + assert.Equal(t, http.StatusOK, res.StatusCode) + assert.Equal(t, "0.1.0", res.Header.Get("X-Api-Version")) + mockRuntime.AssertExpectations(t) +} + +func TestServeChatHealth_UnsupportedVersionHeader(t *testing.T) { + mockRuntime := new(MockRuntime) + + req := httptest.NewRequest(http.MethodGet, "/chat/health", nil) + req.Header.Set("X-Api-Version", "99.0.0") + w := httptest.NewRecorder() + + newMuEdHandler(nil, mockRuntime, "").ServeChatHealth(w, req) + + res := w.Result() + defer res.Body.Close() + raw, _ := io.ReadAll(res.Body) + + assert.Equal(t, http.StatusNotAcceptable, res.StatusCode) + assert.Equal(t, "0.1.0", res.Header.Get("X-Api-Version")) + + var body map[string]any + require.NoError(t, json.Unmarshal(raw, &body)) + assert.Equal(t, "VERSION_NOT_SUPPORTED", body["code"]) + + mockRuntime.AssertNotCalled(t, "ChatHealth", mock.Anything) +} diff --git a/handler/mued.go b/handler/evaluate.go similarity index 94% rename from handler/mued.go rename to handler/evaluate.go index 53c4c73..c462b17 100644 --- a/handler/mued.go +++ b/handler/evaluate.go @@ -10,6 +10,7 @@ import ( "go.uber.org/zap" "github.com/lambda-feedback/shimmy/config" + "github.com/lambda-feedback/shimmy/internal/server" "github.com/lambda-feedback/shimmy/runtime" ) @@ -213,7 +214,7 @@ func (h *MuEdHandler) ServeHealth(w http.ResponseWriter, r *http.Request) { } resp, err := h.runtime.Handle(r.Context(), runtime.EvaluationRequest{ - Command: runtime.CommandHealth, + Command: runtime.CommandEvaluateHealth, Data: map[string]any{}, }) if err != nil { @@ -239,3 +240,11 @@ func (h *MuEdHandler) ServeHealth(w http.ResponseWriter, r *http.Request) { w.WriteHeader(statusCode) json.NewEncoder(w).Encode(result) //nolint:errcheck } + +func NewMuEdEvaluateRoute(handler *MuEdHandler) server.HttpHandlerResult { + return server.AsHttpHandler("/evaluate", http.HandlerFunc(handler.ServeEvaluate)) +} + +func NewMuEdEvaluateHealthRoute(handler *MuEdHandler) server.HttpHandlerResult { + return server.AsHttpHandler("/evaluate/health", http.HandlerFunc(handler.ServeHealth)) +} diff --git a/handler/mued_test.go b/handler/evaluate_test.go similarity index 96% rename from handler/mued_test.go rename to handler/evaluate_test.go index afb65af..5fa03b6 100644 --- a/handler/mued_test.go +++ b/handler/evaluate_test.go @@ -29,6 +29,16 @@ func (m *MockRuntime) Handle(ctx context.Context, req runtime.EvaluationRequest) return args.Get(0).(runtime.EvaluationResponse), args.Error(1) } +func (m *MockRuntime) Chat(ctx context.Context, req runtime.ChatRequest) (runtime.ChatResponse, error) { + args := m.Called(ctx, req) + return args.Get(0).(runtime.ChatResponse), args.Error(1) +} + +func (m *MockRuntime) ChatHealth(ctx context.Context) (runtime.ChatResponse, error) { + args := m.Called(ctx) + return args.Get(0).(runtime.ChatResponse), args.Error(1) +} + func (m *MockRuntime) Start(ctx context.Context) error { return m.Called(ctx).Error(0) } @@ -280,7 +290,7 @@ func TestMuEdServeHealth_Success(t *testing.T) { healthResult := map[string]any{"tests_passed": true, "successes": []any{}, "failures": []any{}, "errors": []any{}} mockRuntime := new(MockRuntime) mockRuntime.On("Handle", mock.Anything, runtime.EvaluationRequest{ - Command: runtime.CommandHealth, + Command: runtime.CommandEvaluateHealth, Data: map[string]any{}, }).Return(runtime.EvaluationResponse{ "command": "healthcheck", @@ -365,7 +375,7 @@ func TestMuEdServeHealth_DegradedStatus(t *testing.T) { healthResult := map[string]any{"tests_passed": false, "successes": []any{}, "failures": []any{"f1"}, "errors": []any{}} mockRuntime := new(MockRuntime) mockRuntime.On("Handle", mock.Anything, runtime.EvaluationRequest{ - Command: runtime.CommandHealth, + Command: runtime.CommandEvaluateHealth, Data: map[string]any{}, }).Return(runtime.EvaluationResponse{ "command": "healthcheck", @@ -456,7 +466,7 @@ func TestMuEdServeHealth_AbsentVersionHeader(t *testing.T) { healthResult := map[string]any{"tests_passed": true, "successes": []any{}, "failures": []any{}, "errors": []any{}} mockRuntime := new(MockRuntime) mockRuntime.On("Handle", mock.Anything, runtime.EvaluationRequest{ - Command: runtime.CommandHealth, + Command: runtime.CommandEvaluateHealth, Data: map[string]any{}, }).Return(runtime.EvaluationResponse{ "command": "healthcheck", diff --git a/handler/module.go b/handler/module.go index a58f29f..c16e381 100644 --- a/handler/module.go +++ b/handler/module.go @@ -10,5 +10,7 @@ func Module() fx.Option { fx.Provide(NewHealthRoute), fx.Provide(NewMuEdEvaluateRoute), fx.Provide(NewMuEdEvaluateHealthRoute), + fx.Provide(NewMuEdChatRoute), + fx.Provide(NewMuEdChatHealthRoute), ) } diff --git a/handler/routes.go b/handler/routes.go index 1419d78..0b26431 100644 --- a/handler/routes.go +++ b/handler/routes.go @@ -13,11 +13,3 @@ func NewLegacyRoute(handler *CommandHandler) server.HttpHandlerResult { func NewHealthRoute() server.HttpHandlerResult { return server.AsHttpHandler("/health", http.HandlerFunc(HealthHandler)) } - -func NewMuEdEvaluateRoute(handler *MuEdHandler) server.HttpHandlerResult { - return server.AsHttpHandler("/evaluate", http.HandlerFunc(handler.ServeEvaluate)) -} - -func NewMuEdEvaluateHealthRoute(handler *MuEdHandler) server.HttpHandlerResult { - return server.AsHttpHandler("/evaluate/health", http.HandlerFunc(handler.ServeHealth)) -} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index 381ad48..b45f80e 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -19,6 +19,12 @@ func NormalizePath(next http.Handler) http.Handler { case strings.HasSuffix(r.URL.Path, "/evaluate"): r = r.Clone(r.Context()) r.URL.Path = "/evaluate" + case strings.HasSuffix(r.URL.Path, "/chat/health"): + r = r.Clone(r.Context()) + r.URL.Path = "/chat/health" + case strings.HasSuffix(r.URL.Path, "/chat"): + r = r.Clone(r.Context()) + r.URL.Path = "/chat" } next.ServeHTTP(w, r) }) diff --git a/internal/server/openapi.go b/internal/server/openapi.go index 3ef2727..8d8ed88 100644 --- a/internal/server/openapi.go +++ b/internal/server/openapi.go @@ -64,6 +64,7 @@ func OpenAPIMiddleware(spec *openapi3.T, log *zap.Logger) (func(http.Handler) ht // Snapshot body before validation — ValidateResponse drains the buffer. bodyBytes := rec.Body.Bytes() + // Validate response (lenient — log only) respInput := &openapi3filter.ResponseValidationInput{ RequestValidationInput: reqInput, diff --git a/runtime/chat.go b/runtime/chat.go new file mode 100644 index 0000000..9c3c93f --- /dev/null +++ b/runtime/chat.go @@ -0,0 +1,141 @@ +package runtime + +import ( + "encoding/json" + "fmt" +) + +// ChatRequest is the dispatcher-level request for the chat command. +type ChatRequest struct { + Data map[string]any +} + +// ChatResponse is the dispatcher-level response for the chat command. +type ChatResponse struct { + Data map[string]any +} + +type MuEdChatRole string + +const ( + MuEdChatRoleUser MuEdChatRole = "USER" + MuEdChatRoleAssistant MuEdChatRole = "ASSISTANT" + MuEdChatRoleSystem MuEdChatRole = "SYSTEM" + MuEdChatRoleTool MuEdChatRole = "TOOL" +) + +type MuEdChatMessage struct { + Role MuEdChatRole `json:"role"` + Content string `json:"content"` +} + +// MuEdChatRequest is the request body for the chat endpoint. Only messages +// and conversationId have a fixed shape per the µEd spec — user, context, +// and configuration are all declared additionalProperties/freeform (or, for +// user, nested under a User schema that isn't worth flattening here), and +// are never inspected by shimmy itself; they only flow straight through to +// the worker. Typing them narrowly risks silently dropping fields that don't +// match a hand-picked sub-schema, so they stay as map[string]any, matching +// the convention used for task-specific data in evaluate.go (e.g. +// MuEdSubmission.Content, MuEdTask.ReferenceSolution). +type MuEdChatRequest struct { + Messages []MuEdChatMessage `json:"messages"` + ConversationID string `json:"conversationId,omitempty"` + User map[string]any `json:"user,omitempty"` + Context map[string]any `json:"context,omitempty"` + Configuration map[string]any `json:"configuration,omitempty"` +} + +type MuEdChatHealthStatus string + +const ( + MuEdChatHealthStatusOK MuEdChatHealthStatus = "OK" + MuEdChatHealthStatusDegraded MuEdChatHealthStatus = "DEGRADED" + MuEdChatHealthStatusUnavailable MuEdChatHealthStatus = "UNAVAILABLE" +) + +// MuEdBuildChatRequest converts a MuEdChatRequest to the map sent to the worker. +func MuEdBuildChatRequest(req MuEdChatRequest) (map[string]any, error) { + if len(req.Messages) == 0 { + return nil, fmt.Errorf("messages must not be empty") + } + b, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("failed to marshal chat request: %w", err) + } + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + return nil, fmt.Errorf("failed to build chat request: %w", err) + } + return m, nil +} + +// MuEdToChatResponse transforms a worker result map into a µEd chat response map. +func MuEdToChatResponse(result map[string]any) (map[string]any, error) { + output, ok := result["output"].(map[string]any) + if !ok { + return nil, fmt.Errorf("chat response missing output") + } + + role, _ := output["role"].(string) + if role == "" { + return nil, fmt.Errorf("chat response missing output role") + } + + content, _ := output["content"].(string) + if content == "" { + return nil, fmt.Errorf("chat response missing output content") + } + + resp := map[string]any{ + "output": map[string]any{ + "role": role, + "content": content, + }, + } + if metadata, ok := result["metadata"].(map[string]any); ok { + resp["metadata"] = metadata + } + return resp, nil +} + +// MuEdToChatHealthResponse transforms a worker result map into a µEd chat +// health response map. Unlike evaluate's health capabilities (which shimmy +// hardcodes itself), a chat worker is authoritative on what it supports, so +// this passes the worker's capabilities through largely as-is — it only +// fills in the spec's required keys/defaults and normalises nil slices to +// empty ones so they serialise as [] not null. +func MuEdToChatHealthResponse(result map[string]any) map[string]any { + status, _ := result["status"].(string) + if status == "" { + status = string(MuEdChatHealthStatusOK) + } + + capabilities, ok := result["capabilities"].(map[string]any) + if !ok { + capabilities = map[string]any{} + } + if _, ok := capabilities["supportsChat"]; !ok { + capabilities["supportsChat"] = false + } + if _, ok := capabilities["supportsDataPolicy"]; !ok { + capabilities["supportsDataPolicy"] = "NOT_SUPPORTED" + } + for _, key := range []string{"supportedLanguages", "supportedModels", "supportedAPIVersions"} { + if capabilities[key] == nil { + capabilities[key] = []string{} + } + } + + resp := map[string]any{ + "status": status, + "capabilities": capabilities, + } + if msg, ok := result["statusMessage"].(string); ok { + resp["statusMessage"] = msg + } + if version, ok := result["version"].(string); ok { + resp["version"] = version + } + return resp +} diff --git a/runtime/chat_test.go b/runtime/chat_test.go new file mode 100644 index 0000000..0e6c79c --- /dev/null +++ b/runtime/chat_test.go @@ -0,0 +1,279 @@ +package runtime_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/lambda-feedback/shimmy/runtime" +) + +// --- MuEdBuildChatRequest --- + +func TestMuEdBuildChatRequest_Valid(t *testing.T) { + req := runtime.MuEdChatRequest{ + Messages: []runtime.MuEdChatMessage{ + {Role: runtime.MuEdChatRoleUser, Content: "hello"}, + }, + } + body, err := runtime.MuEdBuildChatRequest(req) + require.NoError(t, err) + + msgs, ok := body["messages"].([]any) + require.True(t, ok) + require.Len(t, msgs, 1) + msg := msgs[0].(map[string]any) + assert.Equal(t, "USER", msg["role"]) + assert.Equal(t, "hello", msg["content"]) +} + +func TestMuEdBuildChatRequest_EmptyMessages(t *testing.T) { + req := runtime.MuEdChatRequest{Messages: []runtime.MuEdChatMessage{}} + _, err := runtime.MuEdBuildChatRequest(req) + require.Error(t, err) +} + +func TestMuEdBuildChatRequest_NilMessages(t *testing.T) { + req := runtime.MuEdChatRequest{} + _, err := runtime.MuEdBuildChatRequest(req) + require.Error(t, err) +} + +func TestMuEdBuildChatRequest_OptionalFieldsOmitted(t *testing.T) { + req := runtime.MuEdChatRequest{ + Messages: []runtime.MuEdChatMessage{ + {Role: runtime.MuEdChatRoleUser, Content: "hi"}, + }, + } + body, err := runtime.MuEdBuildChatRequest(req) + require.NoError(t, err) + + _, hasUser := body["user"] + _, hasContext := body["context"] + _, hasConversationID := body["conversationId"] + assert.False(t, hasUser) + assert.False(t, hasContext) + assert.False(t, hasConversationID) +} + +func TestMuEdBuildChatRequest_ConversationIDIncluded(t *testing.T) { + req := runtime.MuEdChatRequest{ + Messages: []runtime.MuEdChatMessage{{Role: runtime.MuEdChatRoleUser, Content: "hi"}}, + ConversationID: "abc-123", + } + body, err := runtime.MuEdBuildChatRequest(req) + require.NoError(t, err) + assert.Equal(t, "abc-123", body["conversationId"]) +} + +// TestMuEdBuildChatRequest_UserPassedThroughIntact is a regression test: the +// User field used to be typed as a flat {tone, detail, language} struct that +// didn't match the spec's nested User{type, preference{tone,detail,language}, +// taskProgress} shape, silently discarding everything but the mislabeled +// fields. It must now round-trip untouched, since shimmy never inspects it. +func TestMuEdBuildChatRequest_UserPassedThroughIntact(t *testing.T) { + user := map[string]any{ + "type": "LEARNER", + "preference": map[string]any{ + "tone": "FORMAL", + "conversationalStyle": "socratic", + }, + "taskProgress": map[string]any{ + "timeSpentOnQuestion": "30 minutes", + }, + } + req := runtime.MuEdChatRequest{ + Messages: []runtime.MuEdChatMessage{{Role: runtime.MuEdChatRoleUser, Content: "hi"}}, + User: user, + } + body, err := runtime.MuEdBuildChatRequest(req) + require.NoError(t, err) + + gotUser, ok := body["user"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "LEARNER", gotUser["type"]) + preference, ok := gotUser["preference"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "FORMAL", preference["tone"]) + assert.Equal(t, "socratic", preference["conversationalStyle"]) + taskProgress, ok := gotUser["taskProgress"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "30 minutes", taskProgress["timeSpentOnQuestion"]) +} + +// TestMuEdBuildChatRequest_ContextPassedThroughIntact is a regression test: +// the Context field used to be typed as {course, task, submission}, which +// doesn't match the spec's fully freeform "additionalProperties: true" +// context object. A real caller's context shape (e.g. {set, question, +// summary}) must survive the round trip untouched. +func TestMuEdBuildChatRequest_ContextPassedThroughIntact(t *testing.T) { + context := map[string]any{ + "summary": "prior conversation summary", + "set": map[string]any{ + "title": "Fundamentals", + "number": float64(2), + }, + "question": map[string]any{ + "title": "Understanding Polymorphism", + }, + } + req := runtime.MuEdChatRequest{ + Messages: []runtime.MuEdChatMessage{{Role: runtime.MuEdChatRoleUser, Content: "hi"}}, + Context: context, + } + body, err := runtime.MuEdBuildChatRequest(req) + require.NoError(t, err) + + gotContext, ok := body["context"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "prior conversation summary", gotContext["summary"]) + set, ok := gotContext["set"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "Fundamentals", set["title"]) + question, ok := gotContext["question"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "Understanding Polymorphism", question["title"]) +} + +// --- MuEdToChatResponse --- + +func TestMuEdToChatResponse_Valid(t *testing.T) { + result := map[string]any{ + "output": map[string]any{ + "role": "ASSISTANT", + "content": "Hello there!", + }, + } + resp, err := runtime.MuEdToChatResponse(result) + require.NoError(t, err) + output, ok := resp["output"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "ASSISTANT", output["role"]) + assert.Equal(t, "Hello there!", output["content"]) + assert.NotContains(t, resp, "metadata") +} + +func TestMuEdToChatResponse_MissingOutput(t *testing.T) { + _, err := runtime.MuEdToChatResponse(map[string]any{}) + require.Error(t, err) +} + +func TestMuEdToChatResponse_MissingRole(t *testing.T) { + result := map[string]any{ + "output": map[string]any{ + "content": "Hello", + }, + } + _, err := runtime.MuEdToChatResponse(result) + require.Error(t, err) +} + +func TestMuEdToChatResponse_MissingContent(t *testing.T) { + result := map[string]any{ + "output": map[string]any{ + "role": "ASSISTANT", + }, + } + _, err := runtime.MuEdToChatResponse(result) + require.Error(t, err) +} + +func TestMuEdToChatResponse_MetadataForwarded(t *testing.T) { + result := map[string]any{ + "output": map[string]any{ + "role": "ASSISTANT", + "content": "Hi", + }, + "metadata": map[string]any{ + "model": "gpt-4", + }, + } + resp, err := runtime.MuEdToChatResponse(result) + require.NoError(t, err) + metadata, ok := resp["metadata"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "gpt-4", metadata["model"]) +} + +// --- MuEdToChatHealthResponse --- + +func TestMuEdToChatHealthResponse_Valid(t *testing.T) { + result := map[string]any{ + "status": "DEGRADED", + "capabilities": map[string]any{ + "supportsChat": true, + }, + "statusMessage": "partially degraded", + "version": "1.2.3", + } + resp := runtime.MuEdToChatHealthResponse(result) + assert.Equal(t, "DEGRADED", resp["status"]) + assert.Equal(t, "partially degraded", resp["statusMessage"]) + assert.Equal(t, "1.2.3", resp["version"]) + + capabilities, ok := resp["capabilities"].(map[string]any) + require.True(t, ok) + assert.Equal(t, true, capabilities["supportsChat"]) + // Defaults filled in for required-but-unset spec keys. + assert.Equal(t, "NOT_SUPPORTED", capabilities["supportsDataPolicy"]) + assert.Equal(t, []string{}, capabilities["supportedLanguages"]) + assert.Equal(t, []string{}, capabilities["supportedModels"]) + assert.Equal(t, []string{}, capabilities["supportedAPIVersions"]) +} + +func TestMuEdToChatHealthResponse_CapabilitiesPassedThroughIntact(t *testing.T) { + // The worker is authoritative on its own capabilities (unlike evaluate, + // which hardcodes them) — arbitrary worker-supplied keys must survive. + result := map[string]any{ + "status": "OK", + "capabilities": map[string]any{ + "supportsChat": true, + "supportsUserPreferences": true, + "supportsStreaming": false, + "supportsDataPolicy": "PARTIAL", + "supportedLanguages": []any{"en", "de"}, + "supportedModels": []any{"gpt-4o"}, + "supportedAPIVersions": []any{"0.1.0"}, + }, + } + resp := runtime.MuEdToChatHealthResponse(result) + capabilities, ok := resp["capabilities"].(map[string]any) + require.True(t, ok) + assert.Equal(t, true, capabilities["supportsChat"]) + assert.Equal(t, true, capabilities["supportsUserPreferences"]) + assert.Equal(t, false, capabilities["supportsStreaming"]) + assert.Equal(t, "PARTIAL", capabilities["supportsDataPolicy"]) + assert.Equal(t, []any{"en", "de"}, capabilities["supportedLanguages"]) + assert.Equal(t, []any{"gpt-4o"}, capabilities["supportedModels"]) + assert.Equal(t, []any{"0.1.0"}, capabilities["supportedAPIVersions"]) +} + +func TestMuEdToChatHealthResponse_DefaultsStatusOK(t *testing.T) { + resp := runtime.MuEdToChatHealthResponse(map[string]any{}) + assert.Equal(t, "OK", resp["status"]) +} + +func TestMuEdToChatHealthResponse_DefaultsMissingCapabilities(t *testing.T) { + resp := runtime.MuEdToChatHealthResponse(map[string]any{}) + capabilities, ok := resp["capabilities"].(map[string]any) + require.True(t, ok) + assert.Equal(t, false, capabilities["supportsChat"]) + assert.Equal(t, "NOT_SUPPORTED", capabilities["supportsDataPolicy"]) +} + +func TestMuEdToChatHealthResponse_NilSlicesDefaultToEmpty(t *testing.T) { + resp := runtime.MuEdToChatHealthResponse(map[string]any{}) + + raw, err := json.Marshal(resp) + require.NoError(t, err) + + var out map[string]any + require.NoError(t, json.Unmarshal(raw, &out)) + capabilities, ok := out["capabilities"].(map[string]any) + require.True(t, ok) + assert.Equal(t, []any{}, capabilities["supportedLanguages"]) + assert.Equal(t, []any{}, capabilities["supportedModels"]) + assert.Equal(t, []any{}, capabilities["supportedAPIVersions"]) +} diff --git a/runtime/mued.go b/runtime/evaluate.go similarity index 90% rename from runtime/mued.go rename to runtime/evaluate.go index 24e8b2f..bc9e868 100644 --- a/runtime/mued.go +++ b/runtime/evaluate.go @@ -36,26 +36,6 @@ type MuEdEvaluateRequest struct { PreSubmissionFeedback *MuEdPreSubmissionFeedback `json:"preSubmissionFeedback"` } -var SupportedMuEdVersions = []string{"0.1.0"} - -// MuEdIsVersionSupported reports whether version is in SupportedMuEdVersions. -func MuEdIsVersionSupported(version string) bool { - for _, v := range SupportedMuEdVersions { - if v == version { - return true - } - } - return false -} - -// MuEdResolveVersion returns requested if it's supported, else the latest version. -func MuEdResolveVersion(requested string) string { - if MuEdIsVersionSupported(requested) { - return requested - } - return SupportedMuEdVersions[len(SupportedMuEdVersions)-1] -} - // MuEdToHealthResponse converts a legacy runtime health result to muEd format. func MuEdToHealthResponse(result map[string]any) map[string]any { status := "DEGRADED" diff --git a/runtime/mued_test.go b/runtime/evaluate_test.go similarity index 100% rename from runtime/mued_test.go rename to runtime/evaluate_test.go diff --git a/runtime/handler_test.go b/runtime/handler_test.go index d5dab6d..2ec7836 100644 --- a/runtime/handler_test.go +++ b/runtime/handler_test.go @@ -31,14 +31,20 @@ func (m *mockRuntime) Handle(ctx context.Context, request runtime.EvaluationRequ return args.Get(0).(runtime.EvaluationResponse), args.Error(1) } +func (m *mockRuntime) Chat(ctx context.Context, req runtime.ChatRequest) (runtime.ChatResponse, error) { + panic("not required") +} + +func (m *mockRuntime) ChatHealth(ctx context.Context) (runtime.ChatResponse, error) { + panic("not required") +} + func (m *mockRuntime) Start(ctx context.Context) error { - //Not required for tests - panic("Not required") + panic("not required") } func (m *mockRuntime) Shutdown(ctx context.Context) error { - //Not required for tests - panic("Not required") + panic("not required") } func setupLogger(t *testing.T) *zap.Logger { diff --git a/runtime/handler_validate.go b/runtime/handler_validate.go index e89d6c6..fe6cb43 100644 --- a/runtime/handler_validate.go +++ b/runtime/handler_validate.go @@ -57,7 +57,7 @@ func (r *RuntimeHandler) validate(t validationType, command Command, data map[st zap.Stringer("type", t), ) - if t == validationTypeRequest && command == CommandHealth { + if t == validationTypeRequest && command == CommandEvaluateHealth { // Health does not have a request schema, no need to validate return nil } @@ -94,7 +94,7 @@ func getSchemaType(command Command) (schema.SchemaType, error) { return schema.SchemaTypeEval, nil case CommandPreview: return schema.SchemaTypePreview, nil - case CommandHealth: + case CommandEvaluateHealth: return schema.SchemaTypeHealth, nil default: return 0, errInvalidCommand diff --git a/runtime/models.go b/runtime/models.go index 8e8fa1a..3b81967 100644 --- a/runtime/models.go +++ b/runtime/models.go @@ -14,8 +14,14 @@ const ( // CommandEvaluate is the command to evaluate the response. CommandEvaluate Command = "eval" - // CommandHealth is the command for healthcheck - CommandHealth = "healthcheck" + // CommandEvaluateHealth is the command for healthcheck + CommandEvaluateHealth = "healthcheck" + + // CommandChat is the command for chat. + CommandChat Command = "chat" + + // CommandChatHealth is the command for the chat health check. + CommandChatHealth Command = "chat/health" ) // ParseCommand parses a command from a given path. @@ -26,7 +32,11 @@ func ParseCommand(path string) (Command, bool) { case "preview": return CommandPreview, true case "healthcheck": - return CommandHealth, true + return CommandEvaluateHealth, true + case "chat": + return CommandChat, true + case "chat/health": + return CommandChatHealth, true } return "", false diff --git a/runtime/runtime.go b/runtime/runtime.go index a29ce92..742ee38 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -13,6 +13,9 @@ import ( type Runtime interface { Handle(context.Context, EvaluationRequest) (EvaluationResponse, error) + Chat(context.Context, ChatRequest) (ChatResponse, error) + ChatHealth(context.Context) (ChatResponse, error) + Start(context.Context) error Shutdown(context.Context) error @@ -96,6 +99,16 @@ func (r *EvaluationRuntime) Handle( return r.dispatcher.Send(ctx, string(message.Command), message.Data) } +func (r *EvaluationRuntime) Chat(ctx context.Context, req ChatRequest) (ChatResponse, error) { + data, err := r.dispatcher.Send(ctx, string(CommandChat), req.Data) + return ChatResponse{Data: data}, err +} + +func (r *EvaluationRuntime) ChatHealth(ctx context.Context) (ChatResponse, error) { + data, err := r.dispatcher.Send(ctx, string(CommandChatHealth), map[string]any{}) + return ChatResponse{Data: data}, err +} + func (r *EvaluationRuntime) Shutdown(ctx context.Context) error { return r.dispatcher.Shutdown(ctx) } diff --git a/runtime/version.go b/runtime/version.go new file mode 100644 index 0000000..1763f37 --- /dev/null +++ b/runtime/version.go @@ -0,0 +1,19 @@ +package runtime + +var SupportedMuEdVersions = []string{"0.1.0"} + +func MuEdIsVersionSupported(version string) bool { + for _, v := range SupportedMuEdVersions { + if v == version { + return true + } + } + return false +} + +func MuEdResolveVersion(requested string) string { + if MuEdIsVersionSupported(requested) { + return requested + } + return SupportedMuEdVersions[len(SupportedMuEdVersions)-1] +}