fix: return error instead of panicking on execution engine start failure

cmd/airavata-server/main.go

@@ -166,7 +166,10 @@ func runServer() error {
 		return fmt.Errorf("no workflow backend configured")
 	}
 
-	svcs := app.NewServices(cfg, gdb, repos, workflowBackend)
+	svcs, err := app.NewServices(cfg, gdb, repos, workflowBackend)
+	if err != nil {
+		return fmt.Errorf("build services: %w", err)
+	}
 
 	srv := &http.Server{
 		Addr:              cfg.HTTPAddr,

internal/app/app.go

@@ -159,7 +159,7 @@ func NewRepositories(db *gorm.DB) *Repositories {
 //
 // The repositories stay local: they are an implementation detail of the services above
 // them, and nothing outside this function has ever wanted one directly.
-func NewServices(cfg config.Config, db *gorm.DB, repos *Repositories, workflowBackend backend.Backend) *Services {
+func NewServices(cfg config.Config, db *gorm.DB, repos *Repositories, workflowBackend backend.Backend) (*Services, error) {
 	// Repositories.
 	users := repos.Users
 	groups := repos.Groups
@@ -204,7 +204,10 @@ func NewServices(cfg config.Config, db *gorm.DB, repos *Repositories, workflowBa
 
 	executionEngine := orchestration.NewExecutionEngine(stagingTasks, submissionTasks,
 		monitoringTasks, storages, clusterConfigs, processes, deployments, templates, batchStatus, statuses, workflowBackend)
-	executionEngine.StartEngine()
+
+	if err := executionEngine.StartEngine(); err != nil {
+		return nil, err
+	}
 
 	return &Services{
 		Config: cfg,
@@ -248,5 +251,5 @@ func NewServices(cfg config.Config, db *gorm.DB, repos *Repositories, workflowBa
 			monitoringTasks, commandTasks, executionEngine),
 
 		ExecutionEngine: executionEngine,
-	}
+	}, nil
 }

internal/orchestration/engine.go

@@ -115,10 +115,10 @@ func NewExecutionEngine(dataStagingTasks *processrepo.DataStagingTaskRepository,
 	}
 }
 
-func (w *ExecutionEngine) StartEngine() {
+func (w *ExecutionEngine) StartEngine() error {
 	if w.orchestrator == nil {
 		slog.Info("Execution engine built without a workflow backend; not starting the workflow worker")
-		return
+		return nil
 	}
 
 	slog.Info("Starting execution engine...........")
@@ -131,9 +131,9 @@ func (w *ExecutionEngine) StartEngine() {
 	w.orchestrator.RegisterActivity(w.completeProcess)
 
 	if err := w.orchestrator.Start(ctx); err != nil {
-		slog.Error("Failed to start execution engine", "error", err)
-		panic("Could not start execution engine")
+		return err
 	}
+	return nil
 }
 
 // This is invoked when an email related to a batch job is received.

internal/server/server_test.go

@@ -112,7 +112,10 @@ func newHarness(t *testing.T) *harness {
 
 	repos := app.NewRepositories(gdb)
 	// passing nil for workflow backend to prevent test panic due to sqlite driver collision
-	svcs := app.NewServices(cfg, gdb, repos, nil)
+	svcs, err := app.NewServices(cfg, gdb, repos, nil)
+	if err != nil {
+		t.Fatalf("Failed to initialize services: %v", err)
+	}
 	return &harness{
 		t:    t,
 		db:   gdb,