v313_test.go 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927
  1. package workflow_test
  2. import (
  3. "context"
  4. "testing"
  5. "workflow"
  6. )
  7. // makeV313LLMFileWorkflow returns a v3.13 workflow where an LLM step writes files via out mapping.
  8. func makeV313LLMFileWorkflow(filePaths []string) *workflow.Workflow {
  9. out := workflow.StepOutput{}
  10. for _, p := range filePaths {
  11. out[p] = "=_result"
  12. }
  13. return &workflow.Workflow{
  14. Version: "3.13",
  15. Name: "V313 File Event Test",
  16. Registry: workflow.Registry{
  17. Services: []string{},
  18. Components: []string{},
  19. Vars: []string{},
  20. Files: workflow.FilesRegistry{
  21. Artifacts: []string{"/src/*"},
  22. },
  23. },
  24. Steps: []workflow.Step{
  25. {
  26. ID: "LLM_GenCode",
  27. In: workflow.StepInput{
  28. "messages": []interface{}{
  29. map[string]interface{}{"role": "user", "content": "generate"},
  30. },
  31. },
  32. Out: out,
  33. Next: "Stop_End",
  34. },
  35. {ID: "Stop_End"},
  36. },
  37. }
  38. }
  39. // makeV313WriteStepWorkflow returns a v3.13 workflow with a Write_* step.
  40. func makeV313WriteStepWorkflow() *workflow.Workflow {
  41. return &workflow.Workflow{
  42. Version: "3.13",
  43. Name: "V313 Write Step Test",
  44. Registry: workflow.Registry{
  45. Services: []string{},
  46. Components: []string{},
  47. Vars: []string{},
  48. Files: workflow.FilesRegistry{
  49. Artifacts: []string{"/src/*"},
  50. },
  51. },
  52. Steps: []workflow.Step{
  53. {
  54. ID: "Write_Output",
  55. Target: "/src/output.txt",
  56. Value: "hello world",
  57. Next: "Stop_End",
  58. },
  59. {ID: "Stop_End"},
  60. },
  61. }
  62. }
  63. // ---------------------------------------------------------------------------
  64. // file_start ordering tests (spec 3.13 §13.3)
  65. // ---------------------------------------------------------------------------
  66. // TestFileStartEmittedAfterStepStart verifies that file_start events are emitted
  67. // after step_start and before any llm_token, for an LLM step with file out targets.
  68. func TestFileStartEmittedAfterStepStart(t *testing.T) {
  69. wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
  70. adapters := createTestAdapters()
  71. llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
  72. llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  73. return map[string]interface{}{
  74. "content": "export default function Header() {}",
  75. "model": "gpt-4",
  76. "finish_reason": "stop",
  77. }, nil
  78. })
  79. engine, err := workflow.NewEngine(wf)
  80. if err != nil {
  81. t.Fatalf("NewEngine: %v", err)
  82. }
  83. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  84. if err != nil {
  85. t.Fatalf("Execute: %v", err)
  86. }
  87. var runEvents []workflow.RunEvent
  88. for ev := range result.RunEventStream {
  89. runEvents = append(runEvents, ev)
  90. }
  91. // Find indices for LLM_GenCode
  92. idxStepStart, idxFileStart, idxStepDone := -1, -1, -1
  93. for i, ev := range runEvents {
  94. if ev.StepID != nil && *ev.StepID == "LLM_GenCode" {
  95. switch ev.Type {
  96. case workflow.RunEventStepStart:
  97. idxStepStart = i
  98. case workflow.RunEventFileStart:
  99. if idxFileStart == -1 {
  100. idxFileStart = i
  101. }
  102. case workflow.RunEventStepDone:
  103. idxStepDone = i
  104. }
  105. }
  106. }
  107. if idxStepStart == -1 {
  108. t.Fatal("missing step_start for LLM_GenCode")
  109. }
  110. if idxFileStart == -1 {
  111. t.Fatal("missing file_start for LLM_GenCode")
  112. }
  113. if idxStepDone == -1 {
  114. t.Fatal("missing step_done for LLM_GenCode")
  115. }
  116. // Order: step_start < file_start < step_done
  117. if !(idxStepStart < idxFileStart && idxFileStart < idxStepDone) {
  118. t.Errorf("event order wrong: step_start=%d, file_start=%d, step_done=%d (want start < file_start < done)",
  119. idxStepStart, idxFileStart, idxStepDone)
  120. }
  121. }
  122. // TestFileStartBeforeLLMToken verifies that file_start events precede any llm_token events.
  123. func TestFileStartBeforeLLMToken(t *testing.T) {
  124. wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
  125. // Enable streaming
  126. wf.Steps[0].In["stream"] = true
  127. adapters := createTestAdapters()
  128. llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
  129. llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  130. stream <- "chunk1"
  131. stream <- "chunk2"
  132. return map[string]interface{}{
  133. "content": "chunk1chunk2",
  134. "model": "gpt-4",
  135. "finish_reason": "stop",
  136. }, nil
  137. })
  138. engine, err := workflow.NewEngine(wf)
  139. if err != nil {
  140. t.Fatalf("NewEngine: %v", err)
  141. }
  142. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  143. if err != nil {
  144. t.Fatalf("Execute: %v", err)
  145. }
  146. var runEvents []workflow.RunEvent
  147. for ev := range result.RunEventStream {
  148. runEvents = append(runEvents, ev)
  149. }
  150. // Find first llm_token and last file_start for LLM_GenCode
  151. firstTokenIdx, lastFileStartIdx := -1, -1
  152. for i, ev := range runEvents {
  153. if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
  154. continue
  155. }
  156. if ev.Type == workflow.RunEventLLMToken && firstTokenIdx == -1 {
  157. firstTokenIdx = i
  158. }
  159. if ev.Type == workflow.RunEventFileStart {
  160. lastFileStartIdx = i
  161. }
  162. }
  163. if lastFileStartIdx == -1 {
  164. t.Fatal("missing file_start for LLM_GenCode")
  165. }
  166. if firstTokenIdx == -1 {
  167. t.Fatal("missing llm_token for LLM_GenCode (streaming)")
  168. }
  169. // file_start must come before any llm_token
  170. if lastFileStartIdx >= firstTokenIdx {
  171. t.Errorf("file_start (idx=%d) must precede first llm_token (idx=%d)", lastFileStartIdx, firstTokenIdx)
  172. }
  173. }
  174. // TestFileStartPayloadPathOnly verifies that file_start payload contains only 'path'.
  175. func TestFileStartPayloadPathOnly(t *testing.T) {
  176. wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
  177. adapters := createTestAdapters()
  178. llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
  179. llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  180. return map[string]interface{}{
  181. "content": "code",
  182. "model": "gpt-4",
  183. "finish_reason": "stop",
  184. }, nil
  185. })
  186. engine, err := workflow.NewEngine(wf)
  187. if err != nil {
  188. t.Fatalf("NewEngine: %v", err)
  189. }
  190. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  191. if err != nil {
  192. t.Fatalf("Execute: %v", err)
  193. }
  194. var runEvents []workflow.RunEvent
  195. for ev := range result.RunEventStream {
  196. runEvents = append(runEvents, ev)
  197. }
  198. for _, ev := range runEvents {
  199. if ev.Type != workflow.RunEventFileStart {
  200. continue
  201. }
  202. path, ok := ev.Payload["path"].(string)
  203. if !ok || path == "" {
  204. t.Errorf("file_start payload missing or empty 'path': %v", ev.Payload)
  205. }
  206. // path should not have a leading slash
  207. if len(path) > 0 && path[0] == '/' {
  208. t.Errorf("file_start path should not have leading slash: %q", path)
  209. }
  210. // Should only have 'path' field (spec says payload carries only path)
  211. if len(ev.Payload) != 1 {
  212. t.Errorf("file_start payload should contain only 'path', got: %v", ev.Payload)
  213. }
  214. }
  215. }
  216. // ---------------------------------------------------------------------------
  217. // file_done ordering and payload tests (spec 3.13 §13.3)
  218. // ---------------------------------------------------------------------------
  219. // TestFileDoneAfterLLMDoneBeforeStepDone verifies file_done ordering for LLM steps.
  220. func TestFileDoneAfterLLMDoneBeforeStepDone(t *testing.T) {
  221. wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
  222. adapters := createTestAdapters()
  223. llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
  224. llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  225. return map[string]interface{}{
  226. "content": "export default function Header() {}",
  227. "model": "gpt-4",
  228. "finish_reason": "stop",
  229. }, nil
  230. })
  231. engine, err := workflow.NewEngine(wf)
  232. if err != nil {
  233. t.Fatalf("NewEngine: %v", err)
  234. }
  235. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  236. if err != nil {
  237. t.Fatalf("Execute: %v", err)
  238. }
  239. var runEvents []workflow.RunEvent
  240. for ev := range result.RunEventStream {
  241. runEvents = append(runEvents, ev)
  242. }
  243. idxLLMDone, idxFileDone, idxStepDone := -1, -1, -1
  244. for i, ev := range runEvents {
  245. if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
  246. continue
  247. }
  248. switch ev.Type {
  249. case workflow.RunEventLLMDone:
  250. idxLLMDone = i
  251. case workflow.RunEventFileDone:
  252. if idxFileDone == -1 {
  253. idxFileDone = i
  254. }
  255. case workflow.RunEventStepDone:
  256. idxStepDone = i
  257. }
  258. }
  259. if idxLLMDone == -1 {
  260. t.Fatal("missing llm_done for LLM_GenCode")
  261. }
  262. if idxFileDone == -1 {
  263. t.Fatal("missing file_done for LLM_GenCode")
  264. }
  265. if idxStepDone == -1 {
  266. t.Fatal("missing step_done for LLM_GenCode")
  267. }
  268. // Order: llm_done < file_done < step_done
  269. if !(idxLLMDone < idxFileDone && idxFileDone < idxStepDone) {
  270. t.Errorf("event order wrong: llm_done=%d, file_done=%d, step_done=%d (want llm_done < file_done < step_done)",
  271. idxLLMDone, idxFileDone, idxStepDone)
  272. }
  273. }
  274. // TestFileDonePayload verifies that file_done payload contains 'path' and 'size_bytes'.
  275. func TestFileDonePayload(t *testing.T) {
  276. const fileContent = "export default function Header() { return null; }"
  277. wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
  278. adapters := createTestAdapters()
  279. llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
  280. llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  281. return map[string]interface{}{
  282. "content": fileContent,
  283. "model": "gpt-4",
  284. "finish_reason": "stop",
  285. }, nil
  286. })
  287. engine, err := workflow.NewEngine(wf)
  288. if err != nil {
  289. t.Fatalf("NewEngine: %v", err)
  290. }
  291. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  292. if err != nil {
  293. t.Fatalf("Execute: %v", err)
  294. }
  295. var runEvents []workflow.RunEvent
  296. for ev := range result.RunEventStream {
  297. runEvents = append(runEvents, ev)
  298. }
  299. found := false
  300. for _, ev := range runEvents {
  301. if ev.Type != workflow.RunEventFileDone {
  302. continue
  303. }
  304. found = true
  305. path, ok := ev.Payload["path"].(string)
  306. if !ok || path == "" {
  307. t.Errorf("file_done payload missing 'path': %v", ev.Payload)
  308. }
  309. if len(path) > 0 && path[0] == '/' {
  310. t.Errorf("file_done path should not have leading slash: %q", path)
  311. }
  312. sizeBytes, ok := ev.Payload["size_bytes"]
  313. if !ok {
  314. t.Errorf("file_done payload missing 'size_bytes': %v", ev.Payload)
  315. } else {
  316. size, ok := sizeBytes.(int)
  317. if !ok {
  318. t.Errorf("file_done size_bytes should be int, got %T: %v", sizeBytes, sizeBytes)
  319. } else if size != len([]byte(fileContent)) {
  320. t.Errorf("file_done size_bytes: got %d, want %d", size, len([]byte(fileContent)))
  321. }
  322. }
  323. }
  324. if !found {
  325. t.Error("expected at least one file_done RunEvent, none found")
  326. }
  327. }
  328. // TestMultipleFilesFileStartAndDone verifies that multiple files all get
  329. // file_start and file_done events.
  330. func TestMultipleFilesFileStartAndDone(t *testing.T) {
  331. wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx", "/src/Header.module.css"})
  332. adapters := createTestAdapters()
  333. llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
  334. llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  335. return map[string]interface{}{
  336. "content": "/* styles */",
  337. "model": "gpt-4",
  338. "finish_reason": "stop",
  339. }, nil
  340. })
  341. engine, err := workflow.NewEngine(wf)
  342. if err != nil {
  343. t.Fatalf("NewEngine: %v", err)
  344. }
  345. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  346. if err != nil {
  347. t.Fatalf("Execute: %v", err)
  348. }
  349. var runEvents []workflow.RunEvent
  350. for ev := range result.RunEventStream {
  351. runEvents = append(runEvents, ev)
  352. }
  353. fileStartCount, fileDoneCount := 0, 0
  354. for _, ev := range runEvents {
  355. if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
  356. continue
  357. }
  358. if ev.Type == workflow.RunEventFileStart {
  359. fileStartCount++
  360. }
  361. if ev.Type == workflow.RunEventFileDone {
  362. fileDoneCount++
  363. }
  364. }
  365. if fileStartCount != 2 {
  366. t.Errorf("expected 2 file_start events for 2 files, got %d", fileStartCount)
  367. }
  368. if fileDoneCount != 2 {
  369. t.Errorf("expected 2 file_done events for 2 files, got %d", fileDoneCount)
  370. }
  371. // All file_start events must precede all file_done events
  372. lastFileStartIdx, firstFileDoneIdx := -1, -1
  373. for i, ev := range runEvents {
  374. if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
  375. continue
  376. }
  377. if ev.Type == workflow.RunEventFileStart {
  378. lastFileStartIdx = i
  379. }
  380. if ev.Type == workflow.RunEventFileDone && firstFileDoneIdx == -1 {
  381. firstFileDoneIdx = i
  382. }
  383. }
  384. if lastFileStartIdx == -1 || firstFileDoneIdx == -1 {
  385. t.Fatal("missing file_start or file_done events")
  386. }
  387. if lastFileStartIdx >= firstFileDoneIdx {
  388. t.Errorf("all file_start (last=%d) must precede first file_done (idx=%d)",
  389. lastFileStartIdx, firstFileDoneIdx)
  390. }
  391. }
  392. // ---------------------------------------------------------------------------
  393. // Write_* step file events (spec 3.13 §13.3)
  394. // ---------------------------------------------------------------------------
  395. // TestWriteStepFileEvents verifies that Write_* steps emit file_start and file_done.
  396. func TestWriteStepFileEvents(t *testing.T) {
  397. wf := makeV313WriteStepWorkflow()
  398. adapters := createTestAdapters()
  399. engine, err := workflow.NewEngine(wf)
  400. if err != nil {
  401. t.Fatalf("NewEngine: %v", err)
  402. }
  403. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  404. if err != nil {
  405. t.Fatalf("Execute: %v", err)
  406. }
  407. var runEvents []workflow.RunEvent
  408. for ev := range result.RunEventStream {
  409. runEvents = append(runEvents, ev)
  410. }
  411. // Find file_start and file_done for Write_Output
  412. var fileStartEvs, fileDoneEvs []workflow.RunEvent
  413. idxStepStart, idxStepDone := -1, -1
  414. for i, ev := range runEvents {
  415. if ev.StepID == nil || *ev.StepID != "Write_Output" {
  416. continue
  417. }
  418. switch ev.Type {
  419. case workflow.RunEventStepStart:
  420. idxStepStart = i
  421. case workflow.RunEventFileStart:
  422. fileStartEvs = append(fileStartEvs, ev)
  423. case workflow.RunEventFileDone:
  424. fileDoneEvs = append(fileDoneEvs, ev)
  425. case workflow.RunEventStepDone:
  426. idxStepDone = i
  427. }
  428. }
  429. if len(fileStartEvs) == 0 {
  430. t.Fatal("expected file_start for Write_Output, none found")
  431. }
  432. if len(fileDoneEvs) == 0 {
  433. t.Fatal("expected file_done for Write_Output, none found")
  434. }
  435. // file_start must come after step_start
  436. for _, ev := range fileStartEvs {
  437. if idxStepStart == -1 || ev.Seq <= runEvents[idxStepStart].Seq {
  438. t.Errorf("file_start seq=%d must be after step_start seq=%d", ev.Seq, runEvents[idxStepStart].Seq)
  439. }
  440. }
  441. // file_done must come before step_done
  442. for _, ev := range fileDoneEvs {
  443. if idxStepDone == -1 || ev.Seq >= runEvents[idxStepDone].Seq {
  444. t.Errorf("file_done seq=%d must be before step_done seq=%d", ev.Seq, runEvents[idxStepDone].Seq)
  445. }
  446. }
  447. // Verify file_done has path and size_bytes
  448. for _, ev := range fileDoneEvs {
  449. if _, ok := ev.Payload["path"]; !ok {
  450. t.Errorf("file_done payload missing 'path': %v", ev.Payload)
  451. }
  452. if _, ok := ev.Payload["size_bytes"]; !ok {
  453. t.Errorf("file_done payload missing 'size_bytes': %v", ev.Payload)
  454. }
  455. }
  456. }
  457. // TestNoFileEventsWhenNoFilesWritten verifies that no file_start/file_done events
  458. // are emitted for steps that don't write files.
  459. func TestNoFileEventsWhenNoFilesWritten(t *testing.T) {
  460. wf := &workflow.Workflow{
  461. Version: "3.13",
  462. Name: "No File Events Test",
  463. Registry: workflow.Registry{
  464. Services: []string{},
  465. Components: []string{},
  466. Vars: []string{"$answer(STRING)"},
  467. Files: workflow.FilesRegistry{},
  468. },
  469. Steps: []workflow.Step{
  470. {
  471. ID: "LLM_Answer",
  472. In: workflow.StepInput{
  473. "messages": []interface{}{
  474. map[string]interface{}{"role": "user", "content": "hi"},
  475. },
  476. },
  477. Out: workflow.StepOutput{"$answer": "=_result"},
  478. Next: "Stop_End",
  479. },
  480. {ID: "Stop_End"},
  481. },
  482. }
  483. adapters := createTestAdapters()
  484. llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
  485. llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  486. return map[string]interface{}{"content": "answer", "model": "gpt-4", "finish_reason": "stop"}, nil
  487. })
  488. engine, err := workflow.NewEngine(wf)
  489. if err != nil {
  490. t.Fatalf("NewEngine: %v", err)
  491. }
  492. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  493. if err != nil {
  494. t.Fatalf("Execute: %v", err)
  495. }
  496. for ev := range result.RunEventStream {
  497. if ev.Type == workflow.RunEventFileStart || ev.Type == workflow.RunEventFileDone {
  498. t.Errorf("unexpected %q RunEvent for step with no file outputs", ev.Type)
  499. }
  500. }
  501. }
  502. // ---------------------------------------------------------------------------
  503. // print / step_print tests (spec 3.13 §5.2.12)
  504. // ---------------------------------------------------------------------------
  505. // TestStepPrintLiteralMessage verifies that a literal print value emits step_print
  506. // with the correct message between file_done and step_done.
  507. func TestStepPrintLiteralMessage(t *testing.T) {
  508. wf := &workflow.Workflow{
  509. Version: "3.13",
  510. Name: "Print Literal Test",
  511. Registry: workflow.Registry{
  512. Vars: []string{"$answer(STRING)"},
  513. Files: workflow.FilesRegistry{},
  514. },
  515. Steps: []workflow.Step{
  516. {
  517. ID: "LLM_Answer",
  518. In: workflow.StepInput{
  519. "messages": []interface{}{
  520. map[string]interface{}{"role": "user", "content": "hi"},
  521. },
  522. },
  523. Out: workflow.StepOutput{"$answer": "=_result"},
  524. Print: "step done",
  525. Next: "Stop_End",
  526. },
  527. {ID: "Stop_End"},
  528. },
  529. }
  530. adapters := createTestAdapters()
  531. adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  532. return map[string]interface{}{"content": "hello", "model": "gpt-4", "finish_reason": "stop"}, nil
  533. })
  534. engine, err := workflow.NewEngine(wf)
  535. if err != nil {
  536. t.Fatalf("NewEngine: %v", err)
  537. }
  538. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  539. if err != nil {
  540. t.Fatalf("Execute: %v", err)
  541. }
  542. var runEvents []workflow.RunEvent
  543. for ev := range result.RunEventStream {
  544. runEvents = append(runEvents, ev)
  545. }
  546. // Find step_print event for LLM_Answer
  547. var printEv *workflow.RunEvent
  548. for i := range runEvents {
  549. if runEvents[i].Type == workflow.RunEventStepPrint && runEvents[i].StepID != nil && *runEvents[i].StepID == "LLM_Answer" {
  550. printEv = &runEvents[i]
  551. break
  552. }
  553. }
  554. if printEv == nil {
  555. t.Fatal("expected step_print RunEvent for LLM_Answer, none found")
  556. }
  557. if printEv.Payload["message"] != "step done" {
  558. t.Errorf("step_print message: got %v, want 'step done'", printEv.Payload["message"])
  559. }
  560. }
  561. // TestStepPrintExpressionWithVariable verifies that print evaluates expressions
  562. // and can reference global variables set via out mapping.
  563. func TestStepPrintExpressionWithVariable(t *testing.T) {
  564. wf := &workflow.Workflow{
  565. Version: "3.13",
  566. Name: "Print Expression Test",
  567. Registry: workflow.Registry{
  568. Vars: []string{"$answer(STRING)"},
  569. Files: workflow.FilesRegistry{},
  570. },
  571. Steps: []workflow.Step{
  572. {
  573. ID: "LLM_Answer",
  574. In: workflow.StepInput{
  575. "messages": []interface{}{
  576. map[string]interface{}{"role": "user", "content": "hi"},
  577. },
  578. },
  579. Out: workflow.StepOutput{"$answer": "=_result"},
  580. Print: "=\"Got: \" + $answer",
  581. Next: "Stop_End",
  582. },
  583. {ID: "Stop_End"},
  584. },
  585. }
  586. adapters := createTestAdapters()
  587. adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  588. return map[string]interface{}{"content": "world", "model": "gpt-4", "finish_reason": "stop"}, nil
  589. })
  590. engine, err := workflow.NewEngine(wf)
  591. if err != nil {
  592. t.Fatalf("NewEngine: %v", err)
  593. }
  594. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  595. if err != nil {
  596. t.Fatalf("Execute: %v", err)
  597. }
  598. var runEvents []workflow.RunEvent
  599. for ev := range result.RunEventStream {
  600. runEvents = append(runEvents, ev)
  601. }
  602. var printEv *workflow.RunEvent
  603. for i := range runEvents {
  604. if runEvents[i].Type == workflow.RunEventStepPrint && runEvents[i].StepID != nil && *runEvents[i].StepID == "LLM_Answer" {
  605. printEv = &runEvents[i]
  606. break
  607. }
  608. }
  609. if printEv == nil {
  610. t.Fatal("expected step_print RunEvent, none found")
  611. }
  612. if printEv.Payload["message"] != "Got: world" {
  613. t.Errorf("step_print message: got %v, want 'Got: world'", printEv.Payload["message"])
  614. }
  615. }
  616. // TestStepPrintOrderingBeforeStepDone verifies order: step_print precedes step_done.
  617. func TestStepPrintOrderingBeforeStepDone(t *testing.T) {
  618. wf := &workflow.Workflow{
  619. Version: "3.13",
  620. Name: "Print Order Test",
  621. Registry: workflow.Registry{
  622. Vars: []string{"$answer(STRING)"},
  623. Files: workflow.FilesRegistry{},
  624. },
  625. Steps: []workflow.Step{
  626. {
  627. ID: "LLM_Answer",
  628. In: workflow.StepInput{
  629. "messages": []interface{}{
  630. map[string]interface{}{"role": "user", "content": "hi"},
  631. },
  632. },
  633. Out: workflow.StepOutput{"$answer": "=_result"},
  634. Print: "done",
  635. Next: "Stop_End",
  636. },
  637. {ID: "Stop_End"},
  638. },
  639. }
  640. adapters := createTestAdapters()
  641. adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  642. return map[string]interface{}{"content": "ok", "model": "gpt-4", "finish_reason": "stop"}, nil
  643. })
  644. engine, err := workflow.NewEngine(wf)
  645. if err != nil {
  646. t.Fatalf("NewEngine: %v", err)
  647. }
  648. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  649. if err != nil {
  650. t.Fatalf("Execute: %v", err)
  651. }
  652. var runEvents []workflow.RunEvent
  653. for ev := range result.RunEventStream {
  654. runEvents = append(runEvents, ev)
  655. }
  656. idxPrint, idxDone := -1, -1
  657. for i, ev := range runEvents {
  658. if ev.StepID == nil || *ev.StepID != "LLM_Answer" {
  659. continue
  660. }
  661. if ev.Type == workflow.RunEventStepPrint {
  662. idxPrint = i
  663. }
  664. if ev.Type == workflow.RunEventStepDone {
  665. idxDone = i
  666. }
  667. }
  668. if idxPrint == -1 {
  669. t.Fatal("missing step_print for LLM_Answer")
  670. }
  671. if idxDone == -1 {
  672. t.Fatal("missing step_done for LLM_Answer")
  673. }
  674. if idxPrint >= idxDone {
  675. t.Errorf("step_print (idx=%d) must precede step_done (idx=%d)", idxPrint, idxDone)
  676. }
  677. }
  678. // TestStepPrintNotEmittedWhenSkipped verifies step_print is not emitted when if=false.
  679. func TestStepPrintNotEmittedWhenSkipped(t *testing.T) {
  680. wf := &workflow.Workflow{
  681. Version: "3.13",
  682. Name: "Print Skip Test",
  683. Registry: workflow.Registry{
  684. Vars: []string{"$flag(BOOLEAN)"},
  685. Files: workflow.FilesRegistry{},
  686. },
  687. Steps: []workflow.Step{
  688. {
  689. ID: "LLM_Skipped",
  690. If: "=$flag",
  691. In: workflow.StepInput{"messages": []interface{}{map[string]interface{}{"role": "user", "content": "hi"}}},
  692. Print: "should not appear",
  693. Next: "Stop_End",
  694. },
  695. {ID: "Stop_End"},
  696. },
  697. }
  698. adapters := createTestAdapters()
  699. engine, err := workflow.NewEngine(wf)
  700. if err != nil {
  701. t.Fatalf("NewEngine: %v", err)
  702. }
  703. result, err := engine.Execute(context.Background(), map[string]interface{}{"$flag": false}, adapters)
  704. if err != nil {
  705. t.Fatalf("Execute: %v", err)
  706. }
  707. for ev := range result.RunEventStream {
  708. if ev.Type == workflow.RunEventStepPrint {
  709. t.Errorf("unexpected step_print for skipped step: %v", ev.Payload)
  710. }
  711. }
  712. }
  713. // TestStepPrintNotEmittedWhenFailed verifies step_print is not emitted when the step errors.
  714. func TestStepPrintNotEmittedWhenFailed(t *testing.T) {
  715. wf := &workflow.Workflow{
  716. Version: "3.13",
  717. Name: "Print Fail Test",
  718. Registry: workflow.Registry{
  719. Vars: []string{"$answer(STRING)"},
  720. Files: workflow.FilesRegistry{},
  721. },
  722. Steps: []workflow.Step{
  723. {
  724. ID: "LLM_Fail",
  725. In: workflow.StepInput{"messages": []interface{}{map[string]interface{}{"role": "user", "content": "hi"}}},
  726. Out: workflow.StepOutput{"$answer": "=_result"},
  727. Print: "should not appear",
  728. Next: "Stop_End",
  729. },
  730. {ID: "Stop_End"},
  731. },
  732. }
  733. adapters := createTestAdapters()
  734. adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  735. return nil, &workflow.LLMError{Type: "rate_limit", Message: "limit", Retryable: true}
  736. })
  737. engine, err := workflow.NewEngine(wf)
  738. if err != nil {
  739. t.Fatalf("NewEngine: %v", err)
  740. }
  741. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  742. if err != nil {
  743. t.Fatalf("Execute: %v", err)
  744. }
  745. for ev := range result.RunEventStream {
  746. if ev.Type == workflow.RunEventStepPrint {
  747. t.Errorf("unexpected step_print for failed step: %v", ev.Payload)
  748. }
  749. }
  750. }
  751. // TestStepPrintAfterFileDone verifies order when step has both file outputs and print.
  752. func TestStepPrintAfterFileDone(t *testing.T) {
  753. wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
  754. wf.Steps[0].Print = "files written"
  755. adapters := createTestAdapters()
  756. adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  757. return map[string]interface{}{"content": "code", "model": "gpt-4", "finish_reason": "stop"}, nil
  758. })
  759. engine, err := workflow.NewEngine(wf)
  760. if err != nil {
  761. t.Fatalf("NewEngine: %v", err)
  762. }
  763. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  764. if err != nil {
  765. t.Fatalf("Execute: %v", err)
  766. }
  767. var runEvents []workflow.RunEvent
  768. for ev := range result.RunEventStream {
  769. runEvents = append(runEvents, ev)
  770. }
  771. idxFileDone, idxPrint, idxStepDone := -1, -1, -1
  772. for i, ev := range runEvents {
  773. if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
  774. continue
  775. }
  776. switch ev.Type {
  777. case workflow.RunEventFileDone:
  778. idxFileDone = i
  779. case workflow.RunEventStepPrint:
  780. idxPrint = i
  781. case workflow.RunEventStepDone:
  782. idxStepDone = i
  783. }
  784. }
  785. if idxFileDone == -1 {
  786. t.Fatal("missing file_done")
  787. }
  788. if idxPrint == -1 {
  789. t.Fatal("missing step_print")
  790. }
  791. if idxStepDone == -1 {
  792. t.Fatal("missing step_done")
  793. }
  794. // Order: file_done < step_print < step_done
  795. if !(idxFileDone < idxPrint && idxPrint < idxStepDone) {
  796. t.Errorf("order wrong: file_done=%d, step_print=%d, step_done=%d (want file_done < step_print < step_done)",
  797. idxFileDone, idxPrint, idxStepDone)
  798. }
  799. }
  800. // TestNoPrintOnStopStep verifies that Stop_* steps ignore print (no step_print emitted).
  801. func TestNoPrintOnStopStep(t *testing.T) {
  802. // Stop_* steps can't have print per spec; even if set, the engine should not emit step_print
  803. // (the engine guards with stepType != StepTypeStop)
  804. // We test this indirectly: no step_print should appear for any Stop_* step.
  805. wf := makeV313LLMFileWorkflow([]string{"/src/test.tsx"})
  806. adapters := createTestAdapters()
  807. adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
  808. return map[string]interface{}{"content": "x", "model": "gpt-4", "finish_reason": "stop"}, nil
  809. })
  810. engine, err := workflow.NewEngine(wf)
  811. if err != nil {
  812. t.Fatalf("NewEngine: %v", err)
  813. }
  814. result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
  815. if err != nil {
  816. t.Fatalf("Execute: %v", err)
  817. }
  818. for ev := range result.RunEventStream {
  819. if ev.Type == workflow.RunEventStepPrint && ev.StepID != nil && *ev.StepID == "Stop_End" {
  820. t.Errorf("unexpected step_print for Stop_End step")
  821. }
  822. }
  823. }