response_format_test.go 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965
  1. package workflow
  2. import (
  3. "context"
  4. "encoding/json"
  5. "fmt"
  6. "testing"
  7. )
  8. // TestIsStructuredOutput tests the isStructuredOutput helper function
  9. func TestIsStructuredOutput(t *testing.T) {
  10. tests := []struct {
  11. name string
  12. params map[string]interface{}
  13. expected bool
  14. }{
  15. {
  16. name: "JSONSchemaFormat",
  17. params: map[string]interface{}{
  18. "response_format": map[string]interface{}{
  19. "type": "json_schema",
  20. },
  21. },
  22. expected: true,
  23. },
  24. {
  25. name: "TextFormat",
  26. params: map[string]interface{}{
  27. "response_format": map[string]interface{}{
  28. "type": "text",
  29. },
  30. },
  31. expected: false,
  32. },
  33. {
  34. name: "NoResponseFormat",
  35. params: map[string]interface{}{},
  36. expected: false,
  37. },
  38. {
  39. name: "OutputConfigJSONSchema",
  40. params: map[string]interface{}{
  41. "output_config": map[string]interface{}{
  42. "format": map[string]interface{}{
  43. "type": "json_schema",
  44. },
  45. },
  46. },
  47. expected: true,
  48. },
  49. {
  50. name: "OutputConfigText",
  51. params: map[string]interface{}{
  52. "output_config": map[string]interface{}{
  53. "format": map[string]interface{}{
  54. "type": "text",
  55. },
  56. },
  57. },
  58. expected: false,
  59. },
  60. {
  61. name: "BothFormats",
  62. params: map[string]interface{}{
  63. "response_format": map[string]interface{}{
  64. "type": "json_schema",
  65. },
  66. "output_config": map[string]interface{}{
  67. "format": map[string]interface{}{
  68. "type": "json_schema",
  69. },
  70. },
  71. },
  72. expected: true,
  73. },
  74. }
  75. for _, tt := range tests {
  76. t.Run(tt.name, func(t *testing.T) {
  77. if got := isStructuredOutput(tt.params); got != tt.expected {
  78. t.Errorf("isStructuredOutput() = %v, want %v", got, tt.expected)
  79. }
  80. })
  81. }
  82. }
  83. // TestStructuredOutputIntegration tests the v3.7 structured output feature integration
  84. func TestStructuredOutputIntegration(t *testing.T) {
  85. // Create mock LLM adapter that simulates structured output
  86. mockAdapter := NewDefaultLLMAdapter()
  87. mockJSON := `{"score": 85, "summary": "Good code quality"}`
  88. mockAdapter.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  89. result := map[string]interface{}{
  90. "content": mockJSON,
  91. "model": "gpt-4",
  92. "finish_reason": "stop",
  93. }
  94. // Simulate structured output parsing (like OpenAIAdapter does)
  95. if isStructuredOutput(params) {
  96. var parsed interface{}
  97. if err := json.Unmarshal([]byte(mockJSON), &parsed); err != nil {
  98. return nil, err
  99. }
  100. result["content"] = parsed
  101. }
  102. return result, nil
  103. })
  104. // Create workflow with LLM step using structured output
  105. wf := &Workflow{
  106. Version: "3.7",
  107. Name: "Structured Output Test",
  108. Registry: Registry{
  109. Vars: []string{
  110. "$result(OBJECT)",
  111. },
  112. },
  113. Steps: []Step{
  114. {
  115. ID: "LLM_Test",
  116. In: StepInput{
  117. "model": "gpt-4",
  118. "messages": []interface{}{
  119. map[string]interface{}{
  120. "role": "user",
  121. "content": "Generate a code review",
  122. },
  123. },
  124. "response_format": map[string]interface{}{
  125. "type": "json_schema",
  126. "json_schema": map[string]interface{}{
  127. "name": "code_review",
  128. "description": "A code review response",
  129. "schema": map[string]interface{}{
  130. "type": "object",
  131. "properties": map[string]interface{}{
  132. "score": map[string]interface{}{
  133. "type": "integer",
  134. },
  135. "summary": map[string]interface{}{
  136. "type": "string",
  137. },
  138. },
  139. "required": []interface{}{"score", "summary"},
  140. },
  141. },
  142. },
  143. },
  144. Out: StepOutput{
  145. "$result": "=_result",
  146. },
  147. Next: "Stop_End",
  148. },
  149. {
  150. ID: "Stop_End",
  151. },
  152. },
  153. }
  154. engine, err := NewEngine(wf)
  155. if err != nil {
  156. t.Fatalf("Failed to create engine: %v", err)
  157. }
  158. adapters := &Adapters{
  159. LLM: mockAdapter,
  160. Service: NewDefaultServiceAdapter(),
  161. }
  162. // Execute workflow
  163. result, err := engine.Execute(context.Background(), nil, adapters)
  164. if err != nil {
  165. t.Fatalf("Workflow execution failed: %v", err)
  166. }
  167. // Consume events to wait for workflow completion
  168. for range result.RunEventStream {
  169. }
  170. // Check that $result contains the parsed JSON object
  171. resultVar, ok := result.Context.Variables["$result"]
  172. if !ok {
  173. t.Fatal("$result not found in variables")
  174. }
  175. // Verify it's a parsed object (map), not a string
  176. resultMap, ok := resultVar.(map[string]interface{})
  177. if !ok {
  178. t.Fatalf("Expected $result to be a map[string]interface{}, got %T", resultVar)
  179. }
  180. // Check the parsed fields
  181. if score, ok := resultMap["score"].(float64); !ok || score != 85 {
  182. t.Errorf("Expected score to be 85, got %v", resultMap["score"])
  183. }
  184. if summary, ok := resultMap["summary"].(string); !ok || summary != "Good code quality" {
  185. t.Errorf("Expected summary to be 'Good code quality', got %v", resultMap["summary"])
  186. }
  187. }
  188. // TestOutputConfigIntegration tests the v3.8 output_config direct passthrough feature
  189. func TestOutputConfigIntegration(t *testing.T) {
  190. // Create mock LLM adapter that simulates structured output
  191. mockAdapter := NewDefaultLLMAdapter()
  192. mockJSON := `{"score": 92, "feedback": "Excellent implementation"}`
  193. mockAdapter.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  194. result := map[string]interface{}{
  195. "content": mockJSON,
  196. "model": "claude-sonnet-4-5",
  197. "finish_reason": "stop",
  198. }
  199. // Simulate structured output parsing (like OpenAIAdapter does)
  200. if isStructuredOutput(params) {
  201. var parsed interface{}
  202. if err := json.Unmarshal([]byte(mockJSON), &parsed); err != nil {
  203. return nil, err
  204. }
  205. result["content"] = parsed
  206. }
  207. return result, nil
  208. })
  209. // Create workflow with LLM step using output_config (Anthropic style)
  210. wf := &Workflow{
  211. Version: "3.8",
  212. Name: "Output Config Test",
  213. Registry: Registry{
  214. Vars: []string{
  215. "$result(OBJECT)",
  216. },
  217. },
  218. Steps: []Step{
  219. {
  220. ID: "LLM_Test",
  221. In: StepInput{
  222. "model": "claude-sonnet-4-5",
  223. "messages": []interface{}{
  224. map[string]interface{}{
  225. "role": "user",
  226. "content": "Generate a performance review",
  227. },
  228. },
  229. "output_config": map[string]interface{}{
  230. "format": map[string]interface{}{
  231. "type": "json_schema",
  232. "schema": map[string]interface{}{
  233. "type": "object",
  234. "properties": map[string]interface{}{
  235. "score": map[string]interface{}{
  236. "type": "integer",
  237. },
  238. "feedback": map[string]interface{}{
  239. "type": "string",
  240. },
  241. },
  242. "required": []interface{}{"score", "feedback"},
  243. "additionalProperties": false,
  244. },
  245. },
  246. },
  247. },
  248. Out: StepOutput{
  249. "$result": "=_result",
  250. },
  251. Next: "Stop_End",
  252. },
  253. {
  254. ID: "Stop_End",
  255. },
  256. },
  257. }
  258. engine, err := NewEngine(wf)
  259. if err != nil {
  260. t.Fatalf("Failed to create engine: %v", err)
  261. }
  262. adapters := &Adapters{
  263. LLM: mockAdapter,
  264. Service: NewDefaultServiceAdapter(),
  265. }
  266. // Execute workflow
  267. result, err := engine.Execute(context.Background(), nil, adapters)
  268. if err != nil {
  269. t.Fatalf("Workflow execution failed: %v", err)
  270. }
  271. // Consume events to wait for workflow completion
  272. for range result.RunEventStream {
  273. }
  274. // Check that $result contains the parsed JSON object
  275. resultVar, ok := result.Context.Variables["$result"]
  276. if !ok {
  277. t.Fatal("$result not found in variables")
  278. }
  279. // Verify it's a parsed object (map), not a string
  280. resultMap, ok := resultVar.(map[string]interface{})
  281. if !ok {
  282. t.Fatalf("Expected $result to be a map[string]interface{}, got %T", resultVar)
  283. }
  284. // Check the parsed fields
  285. if score, ok := resultMap["score"].(float64); !ok || score != 92 {
  286. t.Errorf("Expected score to be 92, got %v", resultMap["score"])
  287. }
  288. if feedback, ok := resultMap["feedback"].(string); !ok || feedback != "Excellent implementation" {
  289. t.Errorf("Expected feedback to be 'Excellent implementation', got %v", resultMap["feedback"])
  290. }
  291. }
  292. // TestApplyOutputConfig tests the applyOutputConfig method
  293. func TestApplyOutputConfig(t *testing.T) {
  294. adapter := NewOpenAIAdapter(OpenAIConfig{
  295. BaseURL: "http://localhost:4000",
  296. })
  297. tests := []struct {
  298. name string
  299. outputConfig map[string]interface{}
  300. wantError bool
  301. validate func(t *testing.T, req *ChatCompletionRequest)
  302. }{
  303. {
  304. name: "ValidJSONSchema",
  305. outputConfig: map[string]interface{}{
  306. "format": map[string]interface{}{
  307. "type": "json_schema",
  308. "schema": map[string]interface{}{
  309. "type": "object",
  310. "properties": map[string]interface{}{
  311. "result": map[string]interface{}{"type": "string"},
  312. },
  313. "additionalProperties": false,
  314. },
  315. },
  316. },
  317. wantError: false,
  318. validate: func(t *testing.T, req *ChatCompletionRequest) {
  319. if req.OutputConfig == nil {
  320. t.Error("Expected OutputConfig to be set")
  321. return
  322. }
  323. if req.OutputConfig.Format == nil {
  324. t.Error("Expected OutputConfig.Format to be set")
  325. return
  326. }
  327. if req.OutputConfig.Format.Type != "json_schema" {
  328. t.Errorf("Expected Format.Type = json_schema, got %s", req.OutputConfig.Format.Type)
  329. }
  330. if req.OutputConfig.Format.Schema == nil {
  331. t.Error("Expected Schema to be set")
  332. return
  333. }
  334. },
  335. },
  336. {
  337. name: "TextFormat",
  338. outputConfig: map[string]interface{}{
  339. "format": map[string]interface{}{
  340. "type": "text",
  341. },
  342. },
  343. wantError: false,
  344. validate: func(t *testing.T, req *ChatCompletionRequest) {
  345. // Text format should not set OutputConfig
  346. if req.OutputConfig != nil {
  347. t.Error("Expected OutputConfig to be nil for text format")
  348. }
  349. },
  350. },
  351. {
  352. name: "MissingFormat",
  353. outputConfig: map[string]interface{}{
  354. "type": "json_schema",
  355. },
  356. wantError: true,
  357. },
  358. {
  359. name: "MissingSchema",
  360. outputConfig: map[string]interface{}{
  361. "format": map[string]interface{}{
  362. "type": "json_schema",
  363. },
  364. },
  365. wantError: true,
  366. },
  367. }
  368. for _, tt := range tests {
  369. t.Run(tt.name, func(t *testing.T) {
  370. req := &ChatCompletionRequest{
  371. Model: "claude-sonnet-4-5",
  372. }
  373. err := adapter.applyOutputConfig(req, tt.outputConfig)
  374. if (err != nil) != tt.wantError {
  375. t.Errorf("applyOutputConfig() error = %v, wantError %v", err, tt.wantError)
  376. return
  377. }
  378. if !tt.wantError && tt.validate != nil {
  379. tt.validate(t, req)
  380. }
  381. })
  382. }
  383. }
  384. // TestStructuredOutputToStringVariable tests that structured output (JSON object) is correctly
  385. // marshaled to JSON string when assigned to a STRING variable
  386. func TestStructuredOutputToStringVariable(t *testing.T) {
  387. // Create mock LLM adapter that simulates structured output
  388. mockAdapter := NewDefaultLLMAdapter()
  389. mockJSON := `{"name": "John", "age": 30, "active": true}`
  390. mockAdapter.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  391. result := map[string]interface{}{
  392. "content": mockJSON,
  393. "model": "claude-sonnet-4-5",
  394. "finish_reason": "stop",
  395. }
  396. // Simulate structured output parsing
  397. if isStructuredOutput(params) {
  398. var parsed interface{}
  399. if err := json.Unmarshal([]byte(mockJSON), &parsed); err != nil {
  400. return nil, err
  401. }
  402. result["content"] = parsed
  403. }
  404. return result, nil
  405. })
  406. // Create workflow with LLM step that outputs to a STRING variable
  407. wf := &Workflow{
  408. Version: "3.8",
  409. Name: "Structured Output to String Test",
  410. Registry: Registry{
  411. Vars: []string{
  412. "$userInfo(STRING)", // Declaring as STRING type
  413. },
  414. },
  415. Steps: []Step{
  416. {
  417. ID: "LLM_GetUser",
  418. In: StepInput{
  419. "model": "claude-sonnet-4-5",
  420. "messages": []interface{}{
  421. map[string]interface{}{
  422. "role": "user",
  423. "content": "Get user info",
  424. },
  425. },
  426. "output_config": map[string]interface{}{
  427. "format": map[string]interface{}{
  428. "type": "json_schema",
  429. "schema": map[string]interface{}{
  430. "type": "object",
  431. "properties": map[string]interface{}{
  432. "name": map[string]interface{}{
  433. "type": "string",
  434. },
  435. "age": map[string]interface{}{
  436. "type": "integer",
  437. },
  438. "active": map[string]interface{}{
  439. "type": "boolean",
  440. },
  441. },
  442. "required": []interface{}{"name", "age", "active"},
  443. "additionalProperties": false,
  444. },
  445. },
  446. },
  447. },
  448. Out: StepOutput{
  449. "$userInfo": "=_result", // Assigning JSON object to STRING variable
  450. },
  451. Next: "Stop_End",
  452. },
  453. {
  454. ID: "Stop_End",
  455. },
  456. },
  457. }
  458. engine, err := NewEngine(wf)
  459. if err != nil {
  460. t.Fatalf("Failed to create engine: %v", err)
  461. }
  462. adapters := &Adapters{
  463. LLM: mockAdapter,
  464. Service: NewDefaultServiceAdapter(),
  465. }
  466. // Execute workflow
  467. result, err := engine.Execute(context.Background(), nil, adapters)
  468. if err != nil {
  469. t.Fatalf("Workflow execution failed: %v", err)
  470. }
  471. // Consume events to wait for workflow completion
  472. for range result.RunEventStream {
  473. }
  474. // Check that $userInfo is a STRING (JSON marshaled)
  475. userInfo, ok := result.Context.Variables["$userInfo"]
  476. if !ok {
  477. t.Fatal("$userInfo not found in variables")
  478. }
  479. // Verify it's a string (JSON marshaled from object)
  480. userInfoStr, ok := userInfo.(string)
  481. if !ok {
  482. t.Fatalf("Expected $userInfo to be a string, got %T", userInfo)
  483. }
  484. // Parse the JSON string to verify it's valid JSON
  485. var parsed map[string]interface{}
  486. if err := json.Unmarshal([]byte(userInfoStr), &parsed); err != nil {
  487. t.Fatalf("Expected $userInfo to be valid JSON string, got parse error: %v (value: %q)", err, userInfoStr)
  488. }
  489. // Verify the content is correct
  490. if name, ok := parsed["name"].(string); !ok || name != "John" {
  491. t.Errorf("Expected name to be 'John', got %v", parsed["name"])
  492. }
  493. if age, ok := parsed["age"].(float64); !ok || age != 30 {
  494. t.Errorf("Expected age to be 30, got %v", parsed["age"])
  495. }
  496. if active, ok := parsed["active"].(bool); !ok || active != true {
  497. t.Errorf("Expected active to be true, got %v", parsed["active"])
  498. }
  499. }
  500. // TestVendorParameterMapping tests that response_format is correctly mapped to vendor-specific formats
  501. func TestVendorParameterMapping(t *testing.T) {
  502. tests := []struct {
  503. name string
  504. model string
  505. responseFormat map[string]interface{}
  506. expectOutputConfig bool
  507. expectResponseFormat bool
  508. }{
  509. {
  510. name: "AnthropicModel",
  511. model: "claude-3-5-sonnet-20241022",
  512. responseFormat: map[string]interface{}{
  513. "type": "json_schema",
  514. "json_schema": map[string]interface{}{
  515. "name": "test_schema",
  516. "schema": map[string]interface{}{
  517. "type": "object",
  518. "properties": map[string]interface{}{
  519. "result": map[string]interface{}{"type": "string"},
  520. },
  521. },
  522. },
  523. },
  524. expectOutputConfig: true,
  525. expectResponseFormat: false,
  526. },
  527. {
  528. name: "OpenAIModel",
  529. model: "gpt-4o",
  530. responseFormat: map[string]interface{}{
  531. "type": "json_schema",
  532. "json_schema": map[string]interface{}{
  533. "name": "test_schema",
  534. "schema": map[string]interface{}{
  535. "type": "object",
  536. "properties": map[string]interface{}{
  537. "result": map[string]interface{}{"type": "string"},
  538. },
  539. },
  540. },
  541. },
  542. expectOutputConfig: false,
  543. expectResponseFormat: true,
  544. },
  545. }
  546. for _, tt := range tests {
  547. t.Run(tt.name, func(t *testing.T) {
  548. adapter := NewOpenAIAdapter(OpenAIConfig{
  549. BaseURL: "http://localhost:4000",
  550. })
  551. req := &ChatCompletionRequest{
  552. Model: tt.model,
  553. }
  554. err := adapter.applyResponseFormat(req, tt.responseFormat)
  555. if err != nil {
  556. t.Fatalf("applyResponseFormat failed: %v", err)
  557. }
  558. if tt.expectOutputConfig {
  559. if req.OutputConfig == nil {
  560. t.Error("Expected OutputConfig to be set for Anthropic model")
  561. } else if req.OutputConfig.Format == nil {
  562. t.Error("Expected OutputConfig.Format to be set")
  563. } else if req.OutputConfig.Format.Type != "json_schema" {
  564. t.Errorf("Expected OutputConfig.Format.Type = json_schema, got %s", req.OutputConfig.Format.Type)
  565. }
  566. }
  567. if tt.expectResponseFormat {
  568. if req.ResponseFormat == nil {
  569. t.Error("Expected ResponseFormat to be set for OpenAI model")
  570. } else if req.ResponseFormat.Type != "json_schema" {
  571. t.Errorf("Expected ResponseFormat.Type = json_schema, got %s", req.ResponseFormat.Type)
  572. }
  573. }
  574. })
  575. }
  576. }
  577. // TestSchemaRefResolution tests the v3.9 schemaRef resolution feature
  578. func TestSchemaRefResolution(t *testing.T) {
  579. // Create mock LLM adapter
  580. mockAdapter := NewDefaultLLMAdapter()
  581. mockJSON := `{"projectName": "MyApp", "estimatedDays": 5}`
  582. mockAdapter.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  583. // Verify that schema was resolved (not schemaRef)
  584. if outputConfig, ok := params["output_config"].(map[string]interface{}); ok {
  585. if format, ok := outputConfig["format"].(map[string]interface{}); ok {
  586. if _, hasSchemaRef := format["schemaRef"]; hasSchemaRef {
  587. t.Error("schemaRef should have been resolved before calling LLM adapter")
  588. }
  589. if _, hasSchema := format["schema"]; !hasSchema {
  590. t.Error("schema should be present after schemaRef resolution")
  591. }
  592. }
  593. }
  594. result := map[string]interface{}{
  595. "content": mockJSON,
  596. "model": "claude-sonnet-4-5",
  597. "finish_reason": "stop",
  598. }
  599. if isStructuredOutput(params) {
  600. var parsed interface{}
  601. if err := json.Unmarshal([]byte(mockJSON), &parsed); err != nil {
  602. return nil, err
  603. }
  604. result["content"] = parsed
  605. }
  606. return result, nil
  607. })
  608. // Create workflow with schemaRef
  609. wf := &Workflow{
  610. Version: "3.9",
  611. Name: "SchemaRef Test",
  612. Registry: Registry{
  613. Vars: []string{
  614. "$plan(OBJECT)",
  615. },
  616. Schemas: map[string]map[string]interface{}{
  617. "PlanSchema": {
  618. "type": "object",
  619. "properties": map[string]interface{}{
  620. "projectName": map[string]interface{}{
  621. "type": "string",
  622. },
  623. "estimatedDays": map[string]interface{}{
  624. "type": "integer",
  625. },
  626. },
  627. "required": []interface{}{"projectName", "estimatedDays"},
  628. "additionalProperties": false,
  629. },
  630. },
  631. },
  632. Steps: []Step{
  633. {
  634. ID: "LLM_GeneratePlan",
  635. In: StepInput{
  636. "messages": []interface{}{
  637. map[string]interface{}{
  638. "role": "user",
  639. "content": "Generate a project plan",
  640. },
  641. },
  642. "output_config": map[string]interface{}{
  643. "format": map[string]interface{}{
  644. "type": "json_schema",
  645. "schemaRef": "PlanSchema",
  646. },
  647. },
  648. },
  649. Out: StepOutput{
  650. "$plan": "=_result",
  651. },
  652. Next: "Stop_End",
  653. },
  654. {
  655. ID: "Stop_End",
  656. },
  657. },
  658. }
  659. engine, err := NewEngine(wf)
  660. if err != nil {
  661. t.Fatalf("Failed to create engine: %v", err)
  662. }
  663. adapters := &Adapters{
  664. LLM: mockAdapter,
  665. Service: NewDefaultServiceAdapter(),
  666. }
  667. // Execute workflow
  668. result, err := engine.Execute(context.Background(), nil, adapters)
  669. if err != nil {
  670. t.Fatalf("Workflow execution failed: %v", err)
  671. }
  672. // Consume events
  673. for range result.RunEventStream {
  674. }
  675. // Verify result
  676. planVar, ok := result.Context.Variables["$plan"]
  677. if !ok {
  678. t.Fatal("$plan not found in variables")
  679. }
  680. planMap, ok := planVar.(map[string]interface{})
  681. if !ok {
  682. t.Fatalf("Expected $plan to be a map, got %T", planVar)
  683. }
  684. if projectName, ok := planMap["projectName"].(string); !ok || projectName != "MyApp" {
  685. t.Errorf("Expected projectName to be 'MyApp', got %v", planMap["projectName"])
  686. }
  687. if estimatedDays, ok := planMap["estimatedDays"].(float64); !ok || estimatedDays != 5 {
  688. t.Errorf("Expected estimatedDays to be 5, got %v", planMap["estimatedDays"])
  689. }
  690. }
  691. // TestSchemaRefError tests error handling for invalid schemaRef
  692. func TestSchemaRefError(t *testing.T) {
  693. tests := []struct {
  694. name string
  695. schemas map[string]map[string]interface{}
  696. schemaRef string
  697. expectError string
  698. }{
  699. {
  700. name: "SchemaNotFound",
  701. schemas: map[string]map[string]interface{}{},
  702. schemaRef: "NonExistentSchema",
  703. expectError: "schema not found: NonExistentSchema",
  704. },
  705. {
  706. name: "BothSchemaAndSchemaRef",
  707. schemas: map[string]map[string]interface{}{
  708. "TestSchema": {
  709. "type": "object",
  710. },
  711. },
  712. schemaRef: "", // Will be set in test
  713. expectError: "cannot have both 'schema' and 'schemaRef'",
  714. },
  715. }
  716. for _, tt := range tests {
  717. t.Run(tt.name, func(t *testing.T) {
  718. mockAdapter := NewDefaultLLMAdapter()
  719. outputConfig := map[string]interface{}{
  720. "format": map[string]interface{}{
  721. "type": "json_schema",
  722. },
  723. }
  724. if tt.name == "BothSchemaAndSchemaRef" {
  725. outputConfig["format"].(map[string]interface{})["schema"] = map[string]interface{}{"type": "object"}
  726. outputConfig["format"].(map[string]interface{})["schemaRef"] = "TestSchema"
  727. } else {
  728. outputConfig["format"].(map[string]interface{})["schemaRef"] = tt.schemaRef
  729. }
  730. wf := &Workflow{
  731. Version: "3.9",
  732. Name: "SchemaRef Error Test",
  733. Registry: Registry{
  734. Schemas: tt.schemas,
  735. },
  736. Steps: []Step{
  737. {
  738. ID: "LLM_Test",
  739. In: StepInput{
  740. "messages": []interface{}{},
  741. "output_config": outputConfig,
  742. },
  743. Next: "Stop_End",
  744. },
  745. {
  746. ID: "Stop_End",
  747. },
  748. },
  749. }
  750. engine, err := NewEngine(wf)
  751. if err != nil {
  752. t.Fatalf("Failed to create engine: %v", err)
  753. }
  754. adapters := &Adapters{
  755. LLM: mockAdapter,
  756. Service: NewDefaultServiceAdapter(),
  757. }
  758. result, err := engine.Execute(context.Background(), nil, adapters)
  759. if err != nil {
  760. t.Fatalf("Engine.Execute failed: %v", err)
  761. }
  762. // Consume events and check for error
  763. foundError := false
  764. for event := range result.RunEventStream {
  765. if event.Type == RunEventStepError || event.Type == RunEventWorkflowFailed {
  766. if contains(fmt.Sprintf("%v", event.Payload), tt.expectError) {
  767. foundError = true
  768. }
  769. }
  770. }
  771. if !foundError {
  772. t.Errorf("Expected error containing %q, but no error occurred", tt.expectError)
  773. }
  774. })
  775. }
  776. }
  777. // TestIDEWorkflowValidation tests the v3.9 IDE workflow validation
  778. func TestIDEWorkflowValidation(t *testing.T) {
  779. tests := []struct {
  780. name string
  781. workflow *Workflow
  782. expectError string
  783. }{
  784. {
  785. name: "IDEWorkflowWithServiceNode",
  786. workflow: &Workflow{
  787. Version: "3.9",
  788. Name: "IDE Test",
  789. WorkflowType: WorkflowTypeIDE,
  790. Registry: Registry{},
  791. Steps: []Step{
  792. {
  793. ID: "Service_Test",
  794. Next: "Stop_End",
  795. },
  796. {
  797. ID: "Stop_End",
  798. },
  799. },
  800. },
  801. expectError: "IDE workflow (WorkflowType: IDE) cannot contain Service_* nodes",
  802. },
  803. {
  804. name: "IDEWorkflowWithServicesRegistry",
  805. workflow: &Workflow{
  806. Version: "3.9",
  807. Name: "IDE Test",
  808. WorkflowType: WorkflowTypeIDE,
  809. Registry: Registry{
  810. Services: []string{
  811. "TestService() RETURN result(STRING)",
  812. },
  813. },
  814. Steps: []Step{
  815. {
  816. ID: "LLM_Test",
  817. In: StepInput{},
  818. Next: "Stop_End",
  819. },
  820. {
  821. ID: "Stop_End",
  822. },
  823. },
  824. },
  825. expectError: "IDE workflow (WorkflowType: IDE) must have empty registry.services",
  826. },
  827. {
  828. name: "IDEWorkflowValid",
  829. workflow: &Workflow{
  830. Version: "3.9",
  831. Name: "IDE Test",
  832. WorkflowType: WorkflowTypeIDE,
  833. Registry: Registry{},
  834. Steps: []Step{
  835. {
  836. ID: "LLM_Test",
  837. In: StepInput{},
  838. Next: "Stop_End",
  839. },
  840. {
  841. ID: "Stop_End",
  842. },
  843. },
  844. },
  845. expectError: "",
  846. },
  847. {
  848. name: "BusinessWorkflowWithServiceNode",
  849. workflow: &Workflow{
  850. Version: "3.9",
  851. Name: "Business Test",
  852. WorkflowType: WorkflowTypeBusiness,
  853. Registry: Registry{
  854. Services: []string{
  855. "TestService() RETURN result(STRING)",
  856. },
  857. },
  858. Steps: []Step{
  859. {
  860. ID: "Service_Test",
  861. Next: "Stop_End",
  862. },
  863. {
  864. ID: "Stop_End",
  865. },
  866. },
  867. },
  868. expectError: "",
  869. },
  870. }
  871. for _, tt := range tests {
  872. t.Run(tt.name, func(t *testing.T) {
  873. _, err := NewEngine(tt.workflow)
  874. if tt.expectError == "" {
  875. if err != nil {
  876. t.Errorf("Expected no error, got: %v", err)
  877. }
  878. } else {
  879. if err == nil {
  880. t.Errorf("Expected error containing %q, got no error", tt.expectError)
  881. } else if !contains(err.Error(), tt.expectError) {
  882. t.Errorf("Expected error containing %q, got: %v", tt.expectError, err)
  883. }
  884. }
  885. })
  886. }
  887. }