package workflow import ( "context" "encoding/json" "fmt" "testing" ) // TestIsStructuredOutput tests the isStructuredOutput helper function func TestIsStructuredOutput(t *testing.T) { tests := []struct { name string params map[string]interface{} expected bool }{ { name: "JSONSchemaFormat", params: map[string]interface{}{ "response_format": map[string]interface{}{ "type": "json_schema", }, }, expected: true, }, { name: "TextFormat", params: map[string]interface{}{ "response_format": map[string]interface{}{ "type": "text", }, }, expected: false, }, { name: "NoResponseFormat", params: map[string]interface{}{}, expected: false, }, { name: "OutputConfigJSONSchema", params: map[string]interface{}{ "output_config": map[string]interface{}{ "format": map[string]interface{}{ "type": "json_schema", }, }, }, expected: true, }, { name: "OutputConfigText", params: map[string]interface{}{ "output_config": map[string]interface{}{ "format": map[string]interface{}{ "type": "text", }, }, }, expected: false, }, { name: "BothFormats", params: map[string]interface{}{ "response_format": map[string]interface{}{ "type": "json_schema", }, "output_config": map[string]interface{}{ "format": map[string]interface{}{ "type": "json_schema", }, }, }, expected: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := isStructuredOutput(tt.params); got != tt.expected { t.Errorf("isStructuredOutput() = %v, want %v", got, tt.expected) } }) } } // TestStructuredOutputIntegration tests the v3.7 structured output feature integration func TestStructuredOutputIntegration(t *testing.T) { // Create mock LLM adapter that simulates structured output mockAdapter := NewDefaultLLMAdapter() mockJSON := `{"score": 85, "summary": "Good code quality"}` mockAdapter.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) { result := map[string]interface{}{ "content": mockJSON, "model": "gpt-4", "finish_reason": "stop", } // Simulate structured output parsing (like OpenAIAdapter does) if isStructuredOutput(params) { var parsed interface{} if err := json.Unmarshal([]byte(mockJSON), &parsed); err != nil { return nil, err } result["content"] = parsed } return result, nil }) // Create workflow with LLM step using structured output wf := &Workflow{ Version: "3.7", Name: "Structured Output Test", Registry: Registry{ Vars: []string{ "$result(OBJECT)", }, }, Steps: []Step{ { ID: "LLM_Test", In: StepInput{ "model": "gpt-4", "messages": []interface{}{ map[string]interface{}{ "role": "user", "content": "Generate a code review", }, }, "response_format": map[string]interface{}{ "type": "json_schema", "json_schema": map[string]interface{}{ "name": "code_review", "description": "A code review response", "schema": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ "score": map[string]interface{}{ "type": "integer", }, "summary": map[string]interface{}{ "type": "string", }, }, "required": []interface{}{"score", "summary"}, }, }, }, }, Out: StepOutput{ "$result": "=_result", }, Next: "Stop_End", }, { ID: "Stop_End", }, }, } engine, err := NewEngine(wf) if err != nil { t.Fatalf("Failed to create engine: %v", err) } adapters := &Adapters{ LLM: mockAdapter, Service: NewDefaultServiceAdapter(), } // Execute workflow result, err := engine.Execute(context.Background(), nil, adapters) if err != nil { t.Fatalf("Workflow execution failed: %v", err) } // Consume events to wait for workflow completion for range result.RunEventStream { } // Check that $result contains the parsed JSON object resultVar, ok := result.Context.Variables["$result"] if !ok { t.Fatal("$result not found in variables") } // Verify it's a parsed object (map), not a string resultMap, ok := resultVar.(map[string]interface{}) if !ok { t.Fatalf("Expected $result to be a map[string]interface{}, got %T", resultVar) } // Check the parsed fields if score, ok := resultMap["score"].(float64); !ok || score != 85 { t.Errorf("Expected score to be 85, got %v", resultMap["score"]) } if summary, ok := resultMap["summary"].(string); !ok || summary != "Good code quality" { t.Errorf("Expected summary to be 'Good code quality', got %v", resultMap["summary"]) } } // TestOutputConfigIntegration tests the v3.8 output_config direct passthrough feature func TestOutputConfigIntegration(t *testing.T) { // Create mock LLM adapter that simulates structured output mockAdapter := NewDefaultLLMAdapter() mockJSON := `{"score": 92, "feedback": "Excellent implementation"}` mockAdapter.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) { result := map[string]interface{}{ "content": mockJSON, "model": "claude-sonnet-4-5", "finish_reason": "stop", } // Simulate structured output parsing (like OpenAIAdapter does) if isStructuredOutput(params) { var parsed interface{} if err := json.Unmarshal([]byte(mockJSON), &parsed); err != nil { return nil, err } result["content"] = parsed } return result, nil }) // Create workflow with LLM step using output_config (Anthropic style) wf := &Workflow{ Version: "3.8", Name: "Output Config Test", Registry: Registry{ Vars: []string{ "$result(OBJECT)", }, }, Steps: []Step{ { ID: "LLM_Test", In: StepInput{ "model": "claude-sonnet-4-5", "messages": []interface{}{ map[string]interface{}{ "role": "user", "content": "Generate a performance review", }, }, "output_config": map[string]interface{}{ "format": map[string]interface{}{ "type": "json_schema", "schema": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ "score": map[string]interface{}{ "type": "integer", }, "feedback": map[string]interface{}{ "type": "string", }, }, "required": []interface{}{"score", "feedback"}, "additionalProperties": false, }, }, }, }, Out: StepOutput{ "$result": "=_result", }, Next: "Stop_End", }, { ID: "Stop_End", }, }, } engine, err := NewEngine(wf) if err != nil { t.Fatalf("Failed to create engine: %v", err) } adapters := &Adapters{ LLM: mockAdapter, Service: NewDefaultServiceAdapter(), } // Execute workflow result, err := engine.Execute(context.Background(), nil, adapters) if err != nil { t.Fatalf("Workflow execution failed: %v", err) } // Consume events to wait for workflow completion for range result.RunEventStream { } // Check that $result contains the parsed JSON object resultVar, ok := result.Context.Variables["$result"] if !ok { t.Fatal("$result not found in variables") } // Verify it's a parsed object (map), not a string resultMap, ok := resultVar.(map[string]interface{}) if !ok { t.Fatalf("Expected $result to be a map[string]interface{}, got %T", resultVar) } // Check the parsed fields if score, ok := resultMap["score"].(float64); !ok || score != 92 { t.Errorf("Expected score to be 92, got %v", resultMap["score"]) } if feedback, ok := resultMap["feedback"].(string); !ok || feedback != "Excellent implementation" { t.Errorf("Expected feedback to be 'Excellent implementation', got %v", resultMap["feedback"]) } } // TestApplyOutputConfig tests the applyOutputConfig method func TestApplyOutputConfig(t *testing.T) { adapter := NewOpenAIAdapter(OpenAIConfig{ BaseURL: "http://localhost:4000", }) tests := []struct { name string outputConfig map[string]interface{} wantError bool validate func(t *testing.T, req *ChatCompletionRequest) }{ { name: "ValidJSONSchema", outputConfig: map[string]interface{}{ "format": map[string]interface{}{ "type": "json_schema", "schema": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ "result": map[string]interface{}{"type": "string"}, }, "additionalProperties": false, }, }, }, wantError: false, validate: func(t *testing.T, req *ChatCompletionRequest) { if req.OutputConfig == nil { t.Error("Expected OutputConfig to be set") return } if req.OutputConfig.Format == nil { t.Error("Expected OutputConfig.Format to be set") return } if req.OutputConfig.Format.Type != "json_schema" { t.Errorf("Expected Format.Type = json_schema, got %s", req.OutputConfig.Format.Type) } if req.OutputConfig.Format.Schema == nil { t.Error("Expected Schema to be set") return } }, }, { name: "TextFormat", outputConfig: map[string]interface{}{ "format": map[string]interface{}{ "type": "text", }, }, wantError: false, validate: func(t *testing.T, req *ChatCompletionRequest) { // Text format should not set OutputConfig if req.OutputConfig != nil { t.Error("Expected OutputConfig to be nil for text format") } }, }, { name: "MissingFormat", outputConfig: map[string]interface{}{ "type": "json_schema", }, wantError: true, }, { name: "MissingSchema", outputConfig: map[string]interface{}{ "format": map[string]interface{}{ "type": "json_schema", }, }, wantError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { req := &ChatCompletionRequest{ Model: "claude-sonnet-4-5", } err := adapter.applyOutputConfig(req, tt.outputConfig) if (err != nil) != tt.wantError { t.Errorf("applyOutputConfig() error = %v, wantError %v", err, tt.wantError) return } if !tt.wantError && tt.validate != nil { tt.validate(t, req) } }) } } // TestStructuredOutputToStringVariable tests that structured output (JSON object) is correctly // marshaled to JSON string when assigned to a STRING variable func TestStructuredOutputToStringVariable(t *testing.T) { // Create mock LLM adapter that simulates structured output mockAdapter := NewDefaultLLMAdapter() mockJSON := `{"name": "John", "age": 30, "active": true}` mockAdapter.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) { result := map[string]interface{}{ "content": mockJSON, "model": "claude-sonnet-4-5", "finish_reason": "stop", } // Simulate structured output parsing if isStructuredOutput(params) { var parsed interface{} if err := json.Unmarshal([]byte(mockJSON), &parsed); err != nil { return nil, err } result["content"] = parsed } return result, nil }) // Create workflow with LLM step that outputs to a STRING variable wf := &Workflow{ Version: "3.8", Name: "Structured Output to String Test", Registry: Registry{ Vars: []string{ "$userInfo(STRING)", // Declaring as STRING type }, }, Steps: []Step{ { ID: "LLM_GetUser", In: StepInput{ "model": "claude-sonnet-4-5", "messages": []interface{}{ map[string]interface{}{ "role": "user", "content": "Get user info", }, }, "output_config": map[string]interface{}{ "format": map[string]interface{}{ "type": "json_schema", "schema": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ "name": map[string]interface{}{ "type": "string", }, "age": map[string]interface{}{ "type": "integer", }, "active": map[string]interface{}{ "type": "boolean", }, }, "required": []interface{}{"name", "age", "active"}, "additionalProperties": false, }, }, }, }, Out: StepOutput{ "$userInfo": "=_result", // Assigning JSON object to STRING variable }, Next: "Stop_End", }, { ID: "Stop_End", }, }, } engine, err := NewEngine(wf) if err != nil { t.Fatalf("Failed to create engine: %v", err) } adapters := &Adapters{ LLM: mockAdapter, Service: NewDefaultServiceAdapter(), } // Execute workflow result, err := engine.Execute(context.Background(), nil, adapters) if err != nil { t.Fatalf("Workflow execution failed: %v", err) } // Consume events to wait for workflow completion for range result.RunEventStream { } // Check that $userInfo is a STRING (JSON marshaled) userInfo, ok := result.Context.Variables["$userInfo"] if !ok { t.Fatal("$userInfo not found in variables") } // Verify it's a string (JSON marshaled from object) userInfoStr, ok := userInfo.(string) if !ok { t.Fatalf("Expected $userInfo to be a string, got %T", userInfo) } // Parse the JSON string to verify it's valid JSON var parsed map[string]interface{} if err := json.Unmarshal([]byte(userInfoStr), &parsed); err != nil { t.Fatalf("Expected $userInfo to be valid JSON string, got parse error: %v (value: %q)", err, userInfoStr) } // Verify the content is correct if name, ok := parsed["name"].(string); !ok || name != "John" { t.Errorf("Expected name to be 'John', got %v", parsed["name"]) } if age, ok := parsed["age"].(float64); !ok || age != 30 { t.Errorf("Expected age to be 30, got %v", parsed["age"]) } if active, ok := parsed["active"].(bool); !ok || active != true { t.Errorf("Expected active to be true, got %v", parsed["active"]) } } // TestVendorParameterMapping tests that response_format is correctly mapped to vendor-specific formats func TestVendorParameterMapping(t *testing.T) { tests := []struct { name string model string responseFormat map[string]interface{} expectOutputConfig bool expectResponseFormat bool }{ { name: "AnthropicModel", model: "claude-3-5-sonnet-20241022", responseFormat: map[string]interface{}{ "type": "json_schema", "json_schema": map[string]interface{}{ "name": "test_schema", "schema": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ "result": map[string]interface{}{"type": "string"}, }, }, }, }, expectOutputConfig: true, expectResponseFormat: false, }, { name: "OpenAIModel", model: "gpt-4o", responseFormat: map[string]interface{}{ "type": "json_schema", "json_schema": map[string]interface{}{ "name": "test_schema", "schema": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ "result": map[string]interface{}{"type": "string"}, }, }, }, }, expectOutputConfig: false, expectResponseFormat: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { adapter := NewOpenAIAdapter(OpenAIConfig{ BaseURL: "http://localhost:4000", }) req := &ChatCompletionRequest{ Model: tt.model, } err := adapter.applyResponseFormat(req, tt.responseFormat) if err != nil { t.Fatalf("applyResponseFormat failed: %v", err) } if tt.expectOutputConfig { if req.OutputConfig == nil { t.Error("Expected OutputConfig to be set for Anthropic model") } else if req.OutputConfig.Format == nil { t.Error("Expected OutputConfig.Format to be set") } else if req.OutputConfig.Format.Type != "json_schema" { t.Errorf("Expected OutputConfig.Format.Type = json_schema, got %s", req.OutputConfig.Format.Type) } } if tt.expectResponseFormat { if req.ResponseFormat == nil { t.Error("Expected ResponseFormat to be set for OpenAI model") } else if req.ResponseFormat.Type != "json_schema" { t.Errorf("Expected ResponseFormat.Type = json_schema, got %s", req.ResponseFormat.Type) } } }) } } // TestSchemaRefResolution tests the v3.9 schemaRef resolution feature func TestSchemaRefResolution(t *testing.T) { // Create mock LLM adapter mockAdapter := NewDefaultLLMAdapter() mockJSON := `{"projectName": "MyApp", "estimatedDays": 5}` mockAdapter.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) { // Verify that schema was resolved (not schemaRef) if outputConfig, ok := params["output_config"].(map[string]interface{}); ok { if format, ok := outputConfig["format"].(map[string]interface{}); ok { if _, hasSchemaRef := format["schemaRef"]; hasSchemaRef { t.Error("schemaRef should have been resolved before calling LLM adapter") } if _, hasSchema := format["schema"]; !hasSchema { t.Error("schema should be present after schemaRef resolution") } } } result := map[string]interface{}{ "content": mockJSON, "model": "claude-sonnet-4-5", "finish_reason": "stop", } if isStructuredOutput(params) { var parsed interface{} if err := json.Unmarshal([]byte(mockJSON), &parsed); err != nil { return nil, err } result["content"] = parsed } return result, nil }) // Create workflow with schemaRef wf := &Workflow{ Version: "3.9", Name: "SchemaRef Test", Registry: Registry{ Vars: []string{ "$plan(OBJECT)", }, Schemas: map[string]map[string]interface{}{ "PlanSchema": { "type": "object", "properties": map[string]interface{}{ "projectName": map[string]interface{}{ "type": "string", }, "estimatedDays": map[string]interface{}{ "type": "integer", }, }, "required": []interface{}{"projectName", "estimatedDays"}, "additionalProperties": false, }, }, }, Steps: []Step{ { ID: "LLM_GeneratePlan", In: StepInput{ "messages": []interface{}{ map[string]interface{}{ "role": "user", "content": "Generate a project plan", }, }, "output_config": map[string]interface{}{ "format": map[string]interface{}{ "type": "json_schema", "schemaRef": "PlanSchema", }, }, }, Out: StepOutput{ "$plan": "=_result", }, Next: "Stop_End", }, { ID: "Stop_End", }, }, } engine, err := NewEngine(wf) if err != nil { t.Fatalf("Failed to create engine: %v", err) } adapters := &Adapters{ LLM: mockAdapter, Service: NewDefaultServiceAdapter(), } // Execute workflow result, err := engine.Execute(context.Background(), nil, adapters) if err != nil { t.Fatalf("Workflow execution failed: %v", err) } // Consume events for range result.RunEventStream { } // Verify result planVar, ok := result.Context.Variables["$plan"] if !ok { t.Fatal("$plan not found in variables") } planMap, ok := planVar.(map[string]interface{}) if !ok { t.Fatalf("Expected $plan to be a map, got %T", planVar) } if projectName, ok := planMap["projectName"].(string); !ok || projectName != "MyApp" { t.Errorf("Expected projectName to be 'MyApp', got %v", planMap["projectName"]) } if estimatedDays, ok := planMap["estimatedDays"].(float64); !ok || estimatedDays != 5 { t.Errorf("Expected estimatedDays to be 5, got %v", planMap["estimatedDays"]) } } // TestSchemaRefError tests error handling for invalid schemaRef func TestSchemaRefError(t *testing.T) { tests := []struct { name string schemas map[string]map[string]interface{} schemaRef string expectError string }{ { name: "SchemaNotFound", schemas: map[string]map[string]interface{}{}, schemaRef: "NonExistentSchema", expectError: "schema not found: NonExistentSchema", }, { name: "BothSchemaAndSchemaRef", schemas: map[string]map[string]interface{}{ "TestSchema": { "type": "object", }, }, schemaRef: "", // Will be set in test expectError: "cannot have both 'schema' and 'schemaRef'", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockAdapter := NewDefaultLLMAdapter() outputConfig := map[string]interface{}{ "format": map[string]interface{}{ "type": "json_schema", }, } if tt.name == "BothSchemaAndSchemaRef" { outputConfig["format"].(map[string]interface{})["schema"] = map[string]interface{}{"type": "object"} outputConfig["format"].(map[string]interface{})["schemaRef"] = "TestSchema" } else { outputConfig["format"].(map[string]interface{})["schemaRef"] = tt.schemaRef } wf := &Workflow{ Version: "3.9", Name: "SchemaRef Error Test", Registry: Registry{ Schemas: tt.schemas, }, Steps: []Step{ { ID: "LLM_Test", In: StepInput{ "messages": []interface{}{}, "output_config": outputConfig, }, Next: "Stop_End", }, { ID: "Stop_End", }, }, } engine, err := NewEngine(wf) if err != nil { t.Fatalf("Failed to create engine: %v", err) } adapters := &Adapters{ LLM: mockAdapter, Service: NewDefaultServiceAdapter(), } result, err := engine.Execute(context.Background(), nil, adapters) if err != nil { t.Fatalf("Engine.Execute failed: %v", err) } // Consume events and check for error foundError := false for event := range result.RunEventStream { if event.Type == RunEventStepError || event.Type == RunEventWorkflowFailed { if contains(fmt.Sprintf("%v", event.Payload), tt.expectError) { foundError = true } } } if !foundError { t.Errorf("Expected error containing %q, but no error occurred", tt.expectError) } }) } } // TestIDEWorkflowValidation tests the v3.9 IDE workflow validation func TestIDEWorkflowValidation(t *testing.T) { tests := []struct { name string workflow *Workflow expectError string }{ { name: "IDEWorkflowWithServiceNode", workflow: &Workflow{ Version: "3.9", Name: "IDE Test", WorkflowType: WorkflowTypeIDE, Registry: Registry{}, Steps: []Step{ { ID: "Service_Test", Next: "Stop_End", }, { ID: "Stop_End", }, }, }, expectError: "IDE workflow (WorkflowType: IDE) cannot contain Service_* nodes", }, { name: "IDEWorkflowWithServicesRegistry", workflow: &Workflow{ Version: "3.9", Name: "IDE Test", WorkflowType: WorkflowTypeIDE, Registry: Registry{ Services: []string{ "TestService() RETURN result(STRING)", }, }, Steps: []Step{ { ID: "LLM_Test", In: StepInput{}, Next: "Stop_End", }, { ID: "Stop_End", }, }, }, expectError: "IDE workflow (WorkflowType: IDE) must have empty registry.services", }, { name: "IDEWorkflowValid", workflow: &Workflow{ Version: "3.9", Name: "IDE Test", WorkflowType: WorkflowTypeIDE, Registry: Registry{}, Steps: []Step{ { ID: "LLM_Test", In: StepInput{}, Next: "Stop_End", }, { ID: "Stop_End", }, }, }, expectError: "", }, { name: "BusinessWorkflowWithServiceNode", workflow: &Workflow{ Version: "3.9", Name: "Business Test", WorkflowType: WorkflowTypeBusiness, Registry: Registry{ Services: []string{ "TestService() RETURN result(STRING)", }, }, Steps: []Step{ { ID: "Service_Test", Next: "Stop_End", }, { ID: "Stop_End", }, }, }, expectError: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, err := NewEngine(tt.workflow) if tt.expectError == "" { if err != nil { t.Errorf("Expected no error, got: %v", err) } } else { if err == nil { t.Errorf("Expected error containing %q, got no error", tt.expectError) } else if !contains(err.Error(), tt.expectError) { t.Errorf("Expected error containing %q, got: %v", tt.expectError, err) } } }) } }