| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927 |
- package workflow_test
- import (
- "context"
- "testing"
- "workflow"
- )
- // makeV313LLMFileWorkflow returns a v3.13 workflow where an LLM step writes files via out mapping.
- func makeV313LLMFileWorkflow(filePaths []string) *workflow.Workflow {
- out := workflow.StepOutput{}
- for _, p := range filePaths {
- out[p] = "=_result"
- }
- return &workflow.Workflow{
- Version: "3.13",
- Name: "V313 File Event Test",
- Registry: workflow.Registry{
- Services: []string{},
- Components: []string{},
- Vars: []string{},
- Files: workflow.FilesRegistry{
- Artifacts: []string{"/src/*"},
- },
- },
- Steps: []workflow.Step{
- {
- ID: "LLM_GenCode",
- In: workflow.StepInput{
- "messages": []interface{}{
- map[string]interface{}{"role": "user", "content": "generate"},
- },
- },
- Out: out,
- Next: "Stop_End",
- },
- {ID: "Stop_End"},
- },
- }
- }
- // makeV313WriteStepWorkflow returns a v3.13 workflow with a Write_* step.
- func makeV313WriteStepWorkflow() *workflow.Workflow {
- return &workflow.Workflow{
- Version: "3.13",
- Name: "V313 Write Step Test",
- Registry: workflow.Registry{
- Services: []string{},
- Components: []string{},
- Vars: []string{},
- Files: workflow.FilesRegistry{
- Artifacts: []string{"/src/*"},
- },
- },
- Steps: []workflow.Step{
- {
- ID: "Write_Output",
- Target: "/src/output.txt",
- Value: "hello world",
- Next: "Stop_End",
- },
- {ID: "Stop_End"},
- },
- }
- }
- // ---------------------------------------------------------------------------
- // file_start ordering tests (spec 3.13 §13.3)
- // ---------------------------------------------------------------------------
- // TestFileStartEmittedAfterStepStart verifies that file_start events are emitted
- // after step_start and before any llm_token, for an LLM step with file out targets.
- func TestFileStartEmittedAfterStepStart(t *testing.T) {
- wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
- adapters := createTestAdapters()
- llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
- llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{
- "content": "export default function Header() {}",
- "model": "gpt-4",
- "finish_reason": "stop",
- }, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- // Find indices for LLM_GenCode
- idxStepStart, idxFileStart, idxStepDone := -1, -1, -1
- for i, ev := range runEvents {
- if ev.StepID != nil && *ev.StepID == "LLM_GenCode" {
- switch ev.Type {
- case workflow.RunEventStepStart:
- idxStepStart = i
- case workflow.RunEventFileStart:
- if idxFileStart == -1 {
- idxFileStart = i
- }
- case workflow.RunEventStepDone:
- idxStepDone = i
- }
- }
- }
- if idxStepStart == -1 {
- t.Fatal("missing step_start for LLM_GenCode")
- }
- if idxFileStart == -1 {
- t.Fatal("missing file_start for LLM_GenCode")
- }
- if idxStepDone == -1 {
- t.Fatal("missing step_done for LLM_GenCode")
- }
- // Order: step_start < file_start < step_done
- if !(idxStepStart < idxFileStart && idxFileStart < idxStepDone) {
- t.Errorf("event order wrong: step_start=%d, file_start=%d, step_done=%d (want start < file_start < done)",
- idxStepStart, idxFileStart, idxStepDone)
- }
- }
- // TestFileStartBeforeLLMToken verifies that file_start events precede any llm_token events.
- func TestFileStartBeforeLLMToken(t *testing.T) {
- wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
- // Enable streaming
- wf.Steps[0].In["stream"] = true
- adapters := createTestAdapters()
- llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
- llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- stream <- "chunk1"
- stream <- "chunk2"
- return map[string]interface{}{
- "content": "chunk1chunk2",
- "model": "gpt-4",
- "finish_reason": "stop",
- }, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- // Find first llm_token and last file_start for LLM_GenCode
- firstTokenIdx, lastFileStartIdx := -1, -1
- for i, ev := range runEvents {
- if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
- continue
- }
- if ev.Type == workflow.RunEventLLMToken && firstTokenIdx == -1 {
- firstTokenIdx = i
- }
- if ev.Type == workflow.RunEventFileStart {
- lastFileStartIdx = i
- }
- }
- if lastFileStartIdx == -1 {
- t.Fatal("missing file_start for LLM_GenCode")
- }
- if firstTokenIdx == -1 {
- t.Fatal("missing llm_token for LLM_GenCode (streaming)")
- }
- // file_start must come before any llm_token
- if lastFileStartIdx >= firstTokenIdx {
- t.Errorf("file_start (idx=%d) must precede first llm_token (idx=%d)", lastFileStartIdx, firstTokenIdx)
- }
- }
- // TestFileStartPayloadPathOnly verifies that file_start payload contains only 'path'.
- func TestFileStartPayloadPathOnly(t *testing.T) {
- wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
- adapters := createTestAdapters()
- llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
- llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{
- "content": "code",
- "model": "gpt-4",
- "finish_reason": "stop",
- }, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- for _, ev := range runEvents {
- if ev.Type != workflow.RunEventFileStart {
- continue
- }
- path, ok := ev.Payload["path"].(string)
- if !ok || path == "" {
- t.Errorf("file_start payload missing or empty 'path': %v", ev.Payload)
- }
- // path should not have a leading slash
- if len(path) > 0 && path[0] == '/' {
- t.Errorf("file_start path should not have leading slash: %q", path)
- }
- // Should only have 'path' field (spec says payload carries only path)
- if len(ev.Payload) != 1 {
- t.Errorf("file_start payload should contain only 'path', got: %v", ev.Payload)
- }
- }
- }
- // ---------------------------------------------------------------------------
- // file_done ordering and payload tests (spec 3.13 §13.3)
- // ---------------------------------------------------------------------------
- // TestFileDoneAfterLLMDoneBeforeStepDone verifies file_done ordering for LLM steps.
- func TestFileDoneAfterLLMDoneBeforeStepDone(t *testing.T) {
- wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
- adapters := createTestAdapters()
- llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
- llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{
- "content": "export default function Header() {}",
- "model": "gpt-4",
- "finish_reason": "stop",
- }, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- idxLLMDone, idxFileDone, idxStepDone := -1, -1, -1
- for i, ev := range runEvents {
- if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
- continue
- }
- switch ev.Type {
- case workflow.RunEventLLMDone:
- idxLLMDone = i
- case workflow.RunEventFileDone:
- if idxFileDone == -1 {
- idxFileDone = i
- }
- case workflow.RunEventStepDone:
- idxStepDone = i
- }
- }
- if idxLLMDone == -1 {
- t.Fatal("missing llm_done for LLM_GenCode")
- }
- if idxFileDone == -1 {
- t.Fatal("missing file_done for LLM_GenCode")
- }
- if idxStepDone == -1 {
- t.Fatal("missing step_done for LLM_GenCode")
- }
- // Order: llm_done < file_done < step_done
- if !(idxLLMDone < idxFileDone && idxFileDone < idxStepDone) {
- t.Errorf("event order wrong: llm_done=%d, file_done=%d, step_done=%d (want llm_done < file_done < step_done)",
- idxLLMDone, idxFileDone, idxStepDone)
- }
- }
- // TestFileDonePayload verifies that file_done payload contains 'path' and 'size_bytes'.
- func TestFileDonePayload(t *testing.T) {
- const fileContent = "export default function Header() { return null; }"
- wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
- adapters := createTestAdapters()
- llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
- llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{
- "content": fileContent,
- "model": "gpt-4",
- "finish_reason": "stop",
- }, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- found := false
- for _, ev := range runEvents {
- if ev.Type != workflow.RunEventFileDone {
- continue
- }
- found = true
- path, ok := ev.Payload["path"].(string)
- if !ok || path == "" {
- t.Errorf("file_done payload missing 'path': %v", ev.Payload)
- }
- if len(path) > 0 && path[0] == '/' {
- t.Errorf("file_done path should not have leading slash: %q", path)
- }
- sizeBytes, ok := ev.Payload["size_bytes"]
- if !ok {
- t.Errorf("file_done payload missing 'size_bytes': %v", ev.Payload)
- } else {
- size, ok := sizeBytes.(int)
- if !ok {
- t.Errorf("file_done size_bytes should be int, got %T: %v", sizeBytes, sizeBytes)
- } else if size != len([]byte(fileContent)) {
- t.Errorf("file_done size_bytes: got %d, want %d", size, len([]byte(fileContent)))
- }
- }
- }
- if !found {
- t.Error("expected at least one file_done RunEvent, none found")
- }
- }
- // TestMultipleFilesFileStartAndDone verifies that multiple files all get
- // file_start and file_done events.
- func TestMultipleFilesFileStartAndDone(t *testing.T) {
- wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx", "/src/Header.module.css"})
- adapters := createTestAdapters()
- llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
- llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{
- "content": "/* styles */",
- "model": "gpt-4",
- "finish_reason": "stop",
- }, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- fileStartCount, fileDoneCount := 0, 0
- for _, ev := range runEvents {
- if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
- continue
- }
- if ev.Type == workflow.RunEventFileStart {
- fileStartCount++
- }
- if ev.Type == workflow.RunEventFileDone {
- fileDoneCount++
- }
- }
- if fileStartCount != 2 {
- t.Errorf("expected 2 file_start events for 2 files, got %d", fileStartCount)
- }
- if fileDoneCount != 2 {
- t.Errorf("expected 2 file_done events for 2 files, got %d", fileDoneCount)
- }
- // All file_start events must precede all file_done events
- lastFileStartIdx, firstFileDoneIdx := -1, -1
- for i, ev := range runEvents {
- if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
- continue
- }
- if ev.Type == workflow.RunEventFileStart {
- lastFileStartIdx = i
- }
- if ev.Type == workflow.RunEventFileDone && firstFileDoneIdx == -1 {
- firstFileDoneIdx = i
- }
- }
- if lastFileStartIdx == -1 || firstFileDoneIdx == -1 {
- t.Fatal("missing file_start or file_done events")
- }
- if lastFileStartIdx >= firstFileDoneIdx {
- t.Errorf("all file_start (last=%d) must precede first file_done (idx=%d)",
- lastFileStartIdx, firstFileDoneIdx)
- }
- }
- // ---------------------------------------------------------------------------
- // Write_* step file events (spec 3.13 §13.3)
- // ---------------------------------------------------------------------------
- // TestWriteStepFileEvents verifies that Write_* steps emit file_start and file_done.
- func TestWriteStepFileEvents(t *testing.T) {
- wf := makeV313WriteStepWorkflow()
- adapters := createTestAdapters()
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- // Find file_start and file_done for Write_Output
- var fileStartEvs, fileDoneEvs []workflow.RunEvent
- idxStepStart, idxStepDone := -1, -1
- for i, ev := range runEvents {
- if ev.StepID == nil || *ev.StepID != "Write_Output" {
- continue
- }
- switch ev.Type {
- case workflow.RunEventStepStart:
- idxStepStart = i
- case workflow.RunEventFileStart:
- fileStartEvs = append(fileStartEvs, ev)
- case workflow.RunEventFileDone:
- fileDoneEvs = append(fileDoneEvs, ev)
- case workflow.RunEventStepDone:
- idxStepDone = i
- }
- }
- if len(fileStartEvs) == 0 {
- t.Fatal("expected file_start for Write_Output, none found")
- }
- if len(fileDoneEvs) == 0 {
- t.Fatal("expected file_done for Write_Output, none found")
- }
- // file_start must come after step_start
- for _, ev := range fileStartEvs {
- if idxStepStart == -1 || ev.Seq <= runEvents[idxStepStart].Seq {
- t.Errorf("file_start seq=%d must be after step_start seq=%d", ev.Seq, runEvents[idxStepStart].Seq)
- }
- }
- // file_done must come before step_done
- for _, ev := range fileDoneEvs {
- if idxStepDone == -1 || ev.Seq >= runEvents[idxStepDone].Seq {
- t.Errorf("file_done seq=%d must be before step_done seq=%d", ev.Seq, runEvents[idxStepDone].Seq)
- }
- }
- // Verify file_done has path and size_bytes
- for _, ev := range fileDoneEvs {
- if _, ok := ev.Payload["path"]; !ok {
- t.Errorf("file_done payload missing 'path': %v", ev.Payload)
- }
- if _, ok := ev.Payload["size_bytes"]; !ok {
- t.Errorf("file_done payload missing 'size_bytes': %v", ev.Payload)
- }
- }
- }
- // TestNoFileEventsWhenNoFilesWritten verifies that no file_start/file_done events
- // are emitted for steps that don't write files.
- func TestNoFileEventsWhenNoFilesWritten(t *testing.T) {
- wf := &workflow.Workflow{
- Version: "3.13",
- Name: "No File Events Test",
- Registry: workflow.Registry{
- Services: []string{},
- Components: []string{},
- Vars: []string{"$answer(STRING)"},
- Files: workflow.FilesRegistry{},
- },
- Steps: []workflow.Step{
- {
- ID: "LLM_Answer",
- In: workflow.StepInput{
- "messages": []interface{}{
- map[string]interface{}{"role": "user", "content": "hi"},
- },
- },
- Out: workflow.StepOutput{"$answer": "=_result"},
- Next: "Stop_End",
- },
- {ID: "Stop_End"},
- },
- }
- adapters := createTestAdapters()
- llm := adapters.LLM.(*workflow.DefaultLLMAdapter)
- llm.SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{"content": "answer", "model": "gpt-4", "finish_reason": "stop"}, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- for ev := range result.RunEventStream {
- if ev.Type == workflow.RunEventFileStart || ev.Type == workflow.RunEventFileDone {
- t.Errorf("unexpected %q RunEvent for step with no file outputs", ev.Type)
- }
- }
- }
- // ---------------------------------------------------------------------------
- // print / step_print tests (spec 3.13 §5.2.12)
- // ---------------------------------------------------------------------------
- // TestStepPrintLiteralMessage verifies that a literal print value emits step_print
- // with the correct message between file_done and step_done.
- func TestStepPrintLiteralMessage(t *testing.T) {
- wf := &workflow.Workflow{
- Version: "3.13",
- Name: "Print Literal Test",
- Registry: workflow.Registry{
- Vars: []string{"$answer(STRING)"},
- Files: workflow.FilesRegistry{},
- },
- Steps: []workflow.Step{
- {
- ID: "LLM_Answer",
- In: workflow.StepInput{
- "messages": []interface{}{
- map[string]interface{}{"role": "user", "content": "hi"},
- },
- },
- Out: workflow.StepOutput{"$answer": "=_result"},
- Print: "step done",
- Next: "Stop_End",
- },
- {ID: "Stop_End"},
- },
- }
- adapters := createTestAdapters()
- adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{"content": "hello", "model": "gpt-4", "finish_reason": "stop"}, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- // Find step_print event for LLM_Answer
- var printEv *workflow.RunEvent
- for i := range runEvents {
- if runEvents[i].Type == workflow.RunEventStepPrint && runEvents[i].StepID != nil && *runEvents[i].StepID == "LLM_Answer" {
- printEv = &runEvents[i]
- break
- }
- }
- if printEv == nil {
- t.Fatal("expected step_print RunEvent for LLM_Answer, none found")
- }
- if printEv.Payload["message"] != "step done" {
- t.Errorf("step_print message: got %v, want 'step done'", printEv.Payload["message"])
- }
- }
- // TestStepPrintExpressionWithVariable verifies that print evaluates expressions
- // and can reference global variables set via out mapping.
- func TestStepPrintExpressionWithVariable(t *testing.T) {
- wf := &workflow.Workflow{
- Version: "3.13",
- Name: "Print Expression Test",
- Registry: workflow.Registry{
- Vars: []string{"$answer(STRING)"},
- Files: workflow.FilesRegistry{},
- },
- Steps: []workflow.Step{
- {
- ID: "LLM_Answer",
- In: workflow.StepInput{
- "messages": []interface{}{
- map[string]interface{}{"role": "user", "content": "hi"},
- },
- },
- Out: workflow.StepOutput{"$answer": "=_result"},
- Print: "=\"Got: \" + $answer",
- Next: "Stop_End",
- },
- {ID: "Stop_End"},
- },
- }
- adapters := createTestAdapters()
- adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{"content": "world", "model": "gpt-4", "finish_reason": "stop"}, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- var printEv *workflow.RunEvent
- for i := range runEvents {
- if runEvents[i].Type == workflow.RunEventStepPrint && runEvents[i].StepID != nil && *runEvents[i].StepID == "LLM_Answer" {
- printEv = &runEvents[i]
- break
- }
- }
- if printEv == nil {
- t.Fatal("expected step_print RunEvent, none found")
- }
- if printEv.Payload["message"] != "Got: world" {
- t.Errorf("step_print message: got %v, want 'Got: world'", printEv.Payload["message"])
- }
- }
- // TestStepPrintOrderingBeforeStepDone verifies order: step_print precedes step_done.
- func TestStepPrintOrderingBeforeStepDone(t *testing.T) {
- wf := &workflow.Workflow{
- Version: "3.13",
- Name: "Print Order Test",
- Registry: workflow.Registry{
- Vars: []string{"$answer(STRING)"},
- Files: workflow.FilesRegistry{},
- },
- Steps: []workflow.Step{
- {
- ID: "LLM_Answer",
- In: workflow.StepInput{
- "messages": []interface{}{
- map[string]interface{}{"role": "user", "content": "hi"},
- },
- },
- Out: workflow.StepOutput{"$answer": "=_result"},
- Print: "done",
- Next: "Stop_End",
- },
- {ID: "Stop_End"},
- },
- }
- adapters := createTestAdapters()
- adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{"content": "ok", "model": "gpt-4", "finish_reason": "stop"}, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- idxPrint, idxDone := -1, -1
- for i, ev := range runEvents {
- if ev.StepID == nil || *ev.StepID != "LLM_Answer" {
- continue
- }
- if ev.Type == workflow.RunEventStepPrint {
- idxPrint = i
- }
- if ev.Type == workflow.RunEventStepDone {
- idxDone = i
- }
- }
- if idxPrint == -1 {
- t.Fatal("missing step_print for LLM_Answer")
- }
- if idxDone == -1 {
- t.Fatal("missing step_done for LLM_Answer")
- }
- if idxPrint >= idxDone {
- t.Errorf("step_print (idx=%d) must precede step_done (idx=%d)", idxPrint, idxDone)
- }
- }
- // TestStepPrintNotEmittedWhenSkipped verifies step_print is not emitted when if=false.
- func TestStepPrintNotEmittedWhenSkipped(t *testing.T) {
- wf := &workflow.Workflow{
- Version: "3.13",
- Name: "Print Skip Test",
- Registry: workflow.Registry{
- Vars: []string{"$flag(BOOLEAN)"},
- Files: workflow.FilesRegistry{},
- },
- Steps: []workflow.Step{
- {
- ID: "LLM_Skipped",
- If: "=$flag",
- In: workflow.StepInput{"messages": []interface{}{map[string]interface{}{"role": "user", "content": "hi"}}},
- Print: "should not appear",
- Next: "Stop_End",
- },
- {ID: "Stop_End"},
- },
- }
- adapters := createTestAdapters()
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{"$flag": false}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- for ev := range result.RunEventStream {
- if ev.Type == workflow.RunEventStepPrint {
- t.Errorf("unexpected step_print for skipped step: %v", ev.Payload)
- }
- }
- }
- // TestStepPrintNotEmittedWhenFailed verifies step_print is not emitted when the step errors.
- func TestStepPrintNotEmittedWhenFailed(t *testing.T) {
- wf := &workflow.Workflow{
- Version: "3.13",
- Name: "Print Fail Test",
- Registry: workflow.Registry{
- Vars: []string{"$answer(STRING)"},
- Files: workflow.FilesRegistry{},
- },
- Steps: []workflow.Step{
- {
- ID: "LLM_Fail",
- In: workflow.StepInput{"messages": []interface{}{map[string]interface{}{"role": "user", "content": "hi"}}},
- Out: workflow.StepOutput{"$answer": "=_result"},
- Print: "should not appear",
- Next: "Stop_End",
- },
- {ID: "Stop_End"},
- },
- }
- adapters := createTestAdapters()
- adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return nil, &workflow.LLMError{Type: "rate_limit", Message: "limit", Retryable: true}
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- for ev := range result.RunEventStream {
- if ev.Type == workflow.RunEventStepPrint {
- t.Errorf("unexpected step_print for failed step: %v", ev.Payload)
- }
- }
- }
- // TestStepPrintAfterFileDone verifies order when step has both file outputs and print.
- func TestStepPrintAfterFileDone(t *testing.T) {
- wf := makeV313LLMFileWorkflow([]string{"/src/Header.tsx"})
- wf.Steps[0].Print = "files written"
- adapters := createTestAdapters()
- adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{"content": "code", "model": "gpt-4", "finish_reason": "stop"}, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- var runEvents []workflow.RunEvent
- for ev := range result.RunEventStream {
- runEvents = append(runEvents, ev)
- }
- idxFileDone, idxPrint, idxStepDone := -1, -1, -1
- for i, ev := range runEvents {
- if ev.StepID == nil || *ev.StepID != "LLM_GenCode" {
- continue
- }
- switch ev.Type {
- case workflow.RunEventFileDone:
- idxFileDone = i
- case workflow.RunEventStepPrint:
- idxPrint = i
- case workflow.RunEventStepDone:
- idxStepDone = i
- }
- }
- if idxFileDone == -1 {
- t.Fatal("missing file_done")
- }
- if idxPrint == -1 {
- t.Fatal("missing step_print")
- }
- if idxStepDone == -1 {
- t.Fatal("missing step_done")
- }
- // Order: file_done < step_print < step_done
- if !(idxFileDone < idxPrint && idxPrint < idxStepDone) {
- t.Errorf("order wrong: file_done=%d, step_print=%d, step_done=%d (want file_done < step_print < step_done)",
- idxFileDone, idxPrint, idxStepDone)
- }
- }
- // TestNoPrintOnStopStep verifies that Stop_* steps ignore print (no step_print emitted).
- func TestNoPrintOnStopStep(t *testing.T) {
- // Stop_* steps can't have print per spec; even if set, the engine should not emit step_print
- // (the engine guards with stepType != StepTypeStop)
- // We test this indirectly: no step_print should appear for any Stop_* step.
- wf := makeV313LLMFileWorkflow([]string{"/src/test.tsx"})
- adapters := createTestAdapters()
- adapters.LLM.(*workflow.DefaultLLMAdapter).SetHandler(func(ctx context.Context, params map[string]interface{}, stream chan<- string) (map[string]interface{}, error) {
- return map[string]interface{}{"content": "x", "model": "gpt-4", "finish_reason": "stop"}, nil
- })
- engine, err := workflow.NewEngine(wf)
- if err != nil {
- t.Fatalf("NewEngine: %v", err)
- }
- result, err := engine.Execute(context.Background(), map[string]interface{}{}, adapters)
- if err != nil {
- t.Fatalf("Execute: %v", err)
- }
- for ev := range result.RunEventStream {
- if ev.Type == workflow.RunEventStepPrint && ev.StepID != nil && *ev.StepID == "Stop_End" {
- t.Errorf("unexpected step_print for Stop_End step")
- }
- }
- }
|