Simplify Execute goroutines

core/execute.go

@@ -14,50 +14,19 @@ import (
 	"time"
 )
 
-func readStdout(
+func readPipe(
-	stdout io.ReadCloser,
+	f io.ReadCloser,
-	source string,
+	send chan []byte,
-	postProcess func(item Item) Item,
+	done chan int,
-	items chan Item,
-	cparse chan bool,
 ) {
-	var item Item
+	defer func() {
-	parseError := false
+		done <- 0
-	scanout := bufio.NewScanner(stdout)
+	}()
-	for scanout.Scan() {
+	scanner := bufio.NewScanner(f)
-		data := scanout.Bytes()
+	for scanner.Scan() {
-		err := json.Unmarshal(data, &item)
+		data := scanner.Bytes()
-		if err != nil || item.Id == "" {
+		send <- data
-			log.Printf("[%s: stdout] %s\n", source, strings.TrimSpace(string(data)))
-			parseError = true
-		} else {
-			if postProcess != nil {
-				item = postProcess(item)
 	}
-			item.Active = true   // These fields aren't up to
-			item.Created = 0     // the action to set and
-			item.Source = source // shouldn't be overrideable
-			log.Printf("[%s:   item] %s\n", source, item.Id)
-			items <- item
-		}
-	}
-	// Only send the parsing result at the end, to block main until stdout is drained
-	cparse <- parseError
-	close(items)
-}
-
-func readStderr(stderr io.ReadCloser, source string, done chan bool) {
-	scanerr := bufio.NewScanner(stderr)
-	for scanerr.Scan() {
-		text := strings.TrimSpace(scanerr.Text())
-		log.Printf("[%s: stderr] %s\n", source, text)
-	}
-	done <- true
-}
-
-func writeStdin(stdin io.WriteCloser, text string) {
-	defer stdin.Close()
-	io.WriteString(stdin, text)
 }
 
 func Execute(
@@ -68,7 +37,11 @@ func Execute(
 	input string,
 	timeout time.Duration,
 	postProcess func(item Item) Item,
-) ([]Item, []byte, error) {
+) (
+	items []Item,
+	newState []byte,
+	err error,
+) {
 	log.Printf("executing %v", argv)
 
 	if len(argv) == 0 {
@@ -115,22 +88,14 @@ func Execute(
 		return nil, nil, err
 	}
 
-	cout := make(chan Item)
+	cout := make(chan []byte)
-	cparse := make(chan bool)
+	cerr := make(chan []byte)
-	cerr := make(chan bool)
+	coutDone := make(chan int)
-
+	cerrDone := make(chan int)
-	// Sink routine for items produced
-	var items []Item
-	go func() {
-		for item := range cout {
-			items = append(items, item)
-		}
-	}()
 
 	// Routines handling the process i/o
-	go writeStdin(stdin, input)
+	go readPipe(stdout, cout, coutDone)
-	go readStdout(stdout, source, postProcess, cout, cparse)
+	go readPipe(stderr, cerr, cerrDone)
-	go readStderr(stderr, source, cerr)
 
 	// Kick off the command
 	err = cmd.Start()
@@ -138,9 +103,50 @@ func Execute(
 		return nil, nil, err
 	}
 
-	// Block until std{out,err} close
+	// Write any input to stdin and close it
-	<-cerr
+	io.WriteString(stdin, input)
-	parseError := <-cparse
+	stdin.Close()
+
+	// Collect outputs until std{out,err} close
+	parseError := false
+	stdoutDone := false
+	stderrDone := false
+monitor:
+	for {
+		select {
+		case data := <-cout:
+			var item Item
+			err := json.Unmarshal(data, &item)
+			if err != nil || item.Id == "" {
+				log.Printf("[%s: stdout] %s", source, strings.TrimSpace(string(data)))
+				parseError = true
+			} else {
+				if postProcess != nil {
+					item = postProcess(item)
+				}
+				item.Active = true   // These fields aren't up to
+				item.Created = 0     // the action to set and
+				item.Source = source // shouldn't be overrideable
+				log.Printf("[%s:   item] %s\n", source, item.Id)
+				items = append(items, item)
+			}
+
+		case data := <-cerr:
+			log.Printf("[%s: stderr] %s\n", source, data)
+
+		case <-coutDone:
+			stdoutDone = true
+			if stdoutDone && stderrDone {
+				break monitor
+			}
+
+		case <-cerrDone:
+			stderrDone = true
+			if stdoutDone && stderrDone {
+				break monitor
+			}
+		}
+	}
 
 	err = cmd.Wait()
 	if ctx.Err() == context.DeadlineExceeded {
@@ -159,12 +165,12 @@ func Execute(
 		return nil, nil, errors.New("invalid JSON")
 	}
 
-	newState, err := os.ReadFile(stateFile.Name())
+	newState, err = os.ReadFile(stateFile.Name())
 	if err != nil {
 		return nil, nil, fmt.Errorf("error: failed to read state file: %v", err)
 	}
 
-	return items, newState, nil
+	return
 }
 
 // Execute an action that takes an item as input and returns the item modified.

core/source_test.go

@@ -183,7 +183,7 @@ func TestOnCreateAction(t *testing.T) {
 	items := execute([]string{"jq", "-cn", `{id: "one"}`})
 	add, _, err := update(items)
 	if add != 1 || err != nil {
-		t.Fatal("failed update with noop oncreate")
+		t.Fatalf("failed update with noop oncreate: %v", err)
 	}
 	updated := getItem("one")
 	updated.Created = 0 // zero out for comparison with pre-insert item